evaluation.jsonl

{"run_id": "g4_benchmark", "request_id": "req_R09_rag_llm_r1_72c1a64d", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R09_scrape_llm_r1_6fe4d5a9", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Selected products ['P_THT', 'P_UHEAT', 'P_X4'] not in acceptable selections: [[]]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": ["P_X4", "P_THT", "P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_jev_r1_473ecec1", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_llm_r1_682f97c9", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_rules_r1_45531418", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R02_rag_llm_r1_e63d679f", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R02_scrape_llm_r1_3349545e", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_jev_r1_a0ffe4b5", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 8, "evidence_validity_den": 8, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_rules_r1_61318418", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 15, "evidence_validity_den": 15, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_llm_r1_f268678a", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 39, "evidence_validity_den": 39, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R05_rag_llm_r1_b42e235c", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 7, "evidence_validity_den": 7, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_scrape_llm_r1_46d710fe", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 5, "evidence_validity_den": 5, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_llm_r1_461c164c", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_jev_r1_17045805", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_rules_r1_1123ccaa", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_rag_llm_r1_0aade565", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_scrape_llm_r1_79ae5958", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_llm_r1_d74a92a5", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_rules_r1_17298df8", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_jev_r1_666bca2a", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_rag_llm_r1_061c8c60", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R08_scrape_llm_r1_5533fe5c", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_rules_r1_866e12c7", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_jev_r1_2037f8a3", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R08_RELAY_OUTPUTS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_llm_r1_3064b247", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_X4'] not in acceptable selections: [['P_THT', 'P_X4']]", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R04_rag_llm_r1_a5426bd6", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_RELAY_COUNT'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R04_scrape_llm_r1_0f0f91b8", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_rules_r1_0adfd9c8", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_llm_r1_50eb2525", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_jev_r1_345ed7e5", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Technical verdict 'NOT_APPLICABLE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_RELAY_COUNT'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R07_rag_llm_r1_aa421f82", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R07_CONTROLLER_12DI'", "Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_jev_r1_8d627675", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 2, "evidence_coverage_den": 2, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_scrape_llm_r1_4f550f93", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 2, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R07_CONTROLLER_12DI'", "Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_llm_r1_d04fe757", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_rules_r1_679f3304", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_rag_llm_r1_d3bc0004", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_jev_r1_43b0aba8", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_scrape_llm_r1_30f4f493", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_rules_r1_237c714b", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_llm_r1_a6f0204a", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R06_rag_llm_r1_f1a9ad1a", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_jev_r1_2dca0f0f", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_llm_r1_25955246", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 36, "evidence_validity_den": 36, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_scrape_llm_r1_ef90d45c", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 3, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_rules_r1_af7e6b75", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_rag_llm_r1_afb546e8", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R12_PROFINET_IRT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_jev_r1_7b546cae", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_llm_r1_c40c83c1", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_rules_r1_062e1851", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_scrape_llm_r1_9d5c6617", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 4, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R12_PROFINET_IRT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R10_rag_llm_r1_e6f34ca8", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_jev_r1_f9bd8985", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_rules_r1_758edbf3", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_scrape_llm_r1_60f43c39", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_llm_r1_cdb104c2", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'COMPATIBLE' != gold 'INSUFFICIENT_EVIDENCE'", "Contradiction: Verdict is COMPATIBLE but check 'REQ_SEMANTIC_FUNCTION' is 'FAIL'"], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R01_rag_llm_r1_bca6daac", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 1, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_jev_r1_37f8c7e8", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 1, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 8, "evidence_validity_den": 8, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_rules_r1_13d5ace8", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 1, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_llm_r1_fee2a745", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 1, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 24, "evidence_validity_den": 24, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_scrape_llm_r1_e33ae180", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 1, "order_position": 4, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R12_rag_llm_r2_340c6b4a", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R12_PROFINET_IRT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_scrape_llm_r2_99af35d7", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R12_PROFINET_IRT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_jev_r2_ecf99d98", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_rules_r2_42c66395", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_llm_r2_e5366496", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R09_rag_llm_r2_2edcd766", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R09_scrape_llm_r2_bb902073", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 1, "evidence_validity_den": 1, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Selected products ['P_THT', 'P_UHEAT', 'P_X4'] not in acceptable selections: [[]]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Check 'MODBUS_SENSOR' approved with PASS but has no evidence_ids", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": ["P_X4", "P_THT", "P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_llm_r2_cff07350", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_jev_r2_f250214b", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_rules_r2_43c3388d", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R11_rag_llm_r2_01f552b7", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 1, "evidence_validity_den": 1, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_scrape_llm_r2_0d1c7d6a", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_llm_r2_09247d61", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_rules_r2_397854aa", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_jev_r2_4ee98e3c", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_rag_llm_r2_751fb912", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_RELAY_COUNT'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R04_scrape_llm_r2_57bb5912", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_rules_r2_e1a43e40", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_jev_r2_83fdbc4c", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Technical verdict 'NOT_APPLICABLE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_RELAY_COUNT'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_llm_r2_fe94cccf", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_rag_llm_r2_78edb92b", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_scrape_llm_r2_b5717802", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_rules_r2_c7b8ad57", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_llm_r2_5092d044", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 36, "evidence_validity_den": 36, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_jev_r2_688b357a", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_rag_llm_r2_def50979", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_jev_r2_91ad6d25", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 8, "evidence_validity_den": 8, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_scrape_llm_r2_540d35fc", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_llm_r2_1943529a", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 39, "evidence_validity_den": 39, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_rules_r2_896e5a8a", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 15, "evidence_validity_den": 15, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_rag_llm_r2_8320342b", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 5, "evidence_validity_den": 5, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R08_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_jev_r2_efda7500", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R08_RELAY_OUTPUTS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_scrape_llm_r2_43edeedb", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 2, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_rules_r2_ccea0a19", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_llm_r2_40d4571d", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_X4'] not in acceptable selections: [['P_THT', 'P_X4']]", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R10_rag_llm_r2_4bb1b1d4", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_jev_r2_2488c9ea", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 1, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_llm_r2_5b0ad906", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'COMPATIBLE' != gold 'INSUFFICIENT_EVIDENCE'", "Contradiction: Verdict is COMPATIBLE but check 'REQ_SEMANTIC_FUNCTION' is 'FAIL'"], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R10_scrape_llm_r2_947b1b99", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_rules_r2_a91130f5", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R07_rag_llm_r2_7bb4ee6c", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R07_CONTROLLER_12DI'", "Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_jev_r2_5b21a30e", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 1, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 2, "evidence_coverage_den": 2, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_llm_r2_a9d9e005", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_rules_r2_f60e4c6c", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_scrape_llm_r2_3f360a0e", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 4, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R07_CONTROLLER_12DI'", "Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_rag_llm_r2_6ca1bd5c", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 7, "evidence_validity_den": 7, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_jev_r2_af27f45f", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_rules_r2_60cf590e", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R05_scrape_llm_r2_cc19b588", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 5, "evidence_validity_den": 5, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_llm_r2_7649024e", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R01_rag_llm_r2_48602397", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_jev_r2_8856dcbe", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 8, "evidence_validity_den": 8, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_rules_r2_79884d6e", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_llm_r2_2bc19147", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 24, "evidence_validity_den": 24, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_scrape_llm_r2_cc02da15", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 4, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_rag_llm_r2_c4edb33b", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 2, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_llm_r2_687357b0", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 2, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_scrape_llm_r2_64973742", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 2, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_jev_r2_795533ba", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 2, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_rules_r2_eecf4ea7", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 2, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_rag_llm_r3_44ae841b", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R02_scrape_llm_r3_b42c5a02", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_llm_r3_79c9444f", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 39, "evidence_validity_den": 39, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_jev_r3_19d2040e", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 8, "evidence_validity_den": 8, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R02_structured_rules_r3_438d79b8", "case_id": "R02", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 15, "evidence_validity_den": 15, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R02_TEMP_ACCURACY'", "Missing required check for requirement 'REQ_R02_HUMIDITY_ACCURACY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_THT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_rag_llm_r3_4a4aee3b", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R08_scrape_llm_r3_e47be8c4", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_llm_r3_099a49f3", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_X4'] not in acceptable selections: [['P_THT', 'P_X4']]", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_rules_r3_f81400f2", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R08_SENSOR_MODBUS'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R08_structured_jev_r3_3fa234a3", "case_id": "R08", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R08_RELAY_OUTPUTS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_rag_llm_r3_09a5e100", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R04_RELAY_COUNT'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R04_scrape_llm_r3_fd810e8e", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4"]], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_rules_r3_aa208133", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_jev_r3_de55ff4d", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Technical verdict 'NOT_APPLICABLE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R04_RELAY_COUNT'", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R04_structured_llm_r3_d425bda3", "case_id": "R04", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_X4']]", "Missing required check for requirement 'REQ_R04_SUPPLY_24V'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_rag_llm_r3_aca817a0", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 1, "evidence_validity_den": 1, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_scrape_llm_r3_304bb42b", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 1, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_rules_r3_d74e5199", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_llm_r3_4bd623b4", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R11_structured_jev_r3_126c1b13", "case_id": "R11", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R11_OPTICAL_LASER_CATALOG'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_rag_llm_r3_d37818e2", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_jev_r3_a25d60fb", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_scrape_llm_r3_58a55598", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 2, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_llm_r3_ba1cbebf", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 36, "evidence_validity_den": 36, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INCOMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R06_structured_rules_r3_02e506a5", "case_id": "R06", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R06_HAZARDOUS_CERT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_rag_llm_r3_1ba88dea", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_jev_r3_a0964434", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_scrape_llm_r3_f04cbce9", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_rules_r3_061f2b82", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": true, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R10_structured_llm_r3_85a1b1eb", "case_id": "R10", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 0, "evidence_support": null, "evidence_coverage_num": 0, "evidence_coverage_den": 0, "evidence_coverage": null, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'COMPATIBLE' != gold 'INSUFFICIENT_EVIDENCE'", "Contradiction: Verdict is COMPATIBLE but check 'REQ_SEMANTIC_FUNCTION' is 'FAIL'"], "details": {"selection_applicable": false, "gold_verdict": "INSUFFICIENT_EVIDENCE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R01_rag_llm_r3_84c847a8", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_jev_r3_211897d7", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 8, "evidence_validity_den": 8, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_llm_r3_0ff9e49f", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 24, "evidence_validity_den": 24, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R01_scrape_llm_r3_8a293bcc", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 3, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Missing required check for requirement 'REQ_R01_RELAY_OUTPUTS'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R01_structured_rules_r3_65d7df76", "case_id": "R01", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R01_SUPPLY_VOLTAGE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R03_rag_llm_r3_673f9ba8", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_jev_r3_d7c992b8", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_llm_r3_79ef8574", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R03_structured_rules_r3_b91caca2", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R03_scrape_llm_r3_294515b7", "case_id": "R03", "scenario_family_id": "FAMILY-01-DATA", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R03_HORIZ_WATTAGE'", "Missing required check for requirement 'REQ_R03_VERT_WATTAGE'"], "details": {"selection_applicable": false, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_UHEAT"], []], "resp_selected": ["P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_rag_llm_r3_20f92315", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [['P_THT', 'P_X4']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R07_CONTROLLER_12DI'", "Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_jev_r3_04a9ead0", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": true, "content_fully_correct": true, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": true, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 2, "evidence_coverage_den": 2, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": [], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_rules_r3_4cf1b81c", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_scrape_llm_r3_76a4104e", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 3, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 1, "evidence_validity_den": 1, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R07_CONTROLLER_12DI'", "Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'", "Check 'SENSOR_MODBUS_RTU_TEMP_HUM' approved with PASS but has no evidence_ids", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R07_structured_llm_r3_eafb3d03", "case_id": "R07", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R07_SENSOR_MODBUS'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_X4", "P_THT"]], "resp_selected": ["P_X4", "P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R12_rag_llm_r3_ef214a62", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 2, "evidence_validity_den": 2, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R12_PROFINET_IRT'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_jev_r3_b4d964ce", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_rules_r3_101eccf2", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_structured_llm_r3_beb043f5", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 1, "evidence_coverage_den": 1, "evidence_coverage": 1.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'NOT_APPLICABLE' != gold 'INCOMPATIBLE'", "Check 'REQ_R12_PROFINET_IRT' has status 'PASS', expected 'FAIL'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "NOT_APPLICABLE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R12_scrape_llm_r3_c517908c", "case_id": "R12", "scenario_family_id": "FAMILY-04-UNCERTAINTY", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 4, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Technical verdict 'UNDETERMINED' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R12_PROFINET_IRT'"], "details": {"selection_applicable": false, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "UNDETERMINED", "gold_acceptable": [], "resp_selected": ["P_X4"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_rag_llm_r3_30fc4b3e", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_llm_r3_ff780ad5", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_scrape_llm_r3_a22412e2", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 2, "execution_status": "budget_exceeded", "task_success": false, "content_fully_correct": false, "operationally_valid": false, "execution_completed": false, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Execution status is 'budget_exceeded', expected 'completed'", "Selected products ['P_THT', 'P_UHEAT', 'P_X4'] not in acceptable selections: [[]]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [], "resp_selected": ["P_X4", "P_THT", "P_UHEAT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_jev_r3_b4e456c5", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R09_structured_rules_r3_c8f8e39a", "case_id": "R09", "scenario_family_id": "FAMILY-03-MULTI", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": true, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 4, "evidence_validity_den": 4, "evidence_validity": 1.0, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 1, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products ['P_THT'] not in acceptable selections: [[]]", "Technical verdict 'COMPATIBLE' != gold 'INCOMPATIBLE'", "Missing required check for requirement 'REQ_R09_HEATER_480V_3PHASE'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "INCOMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_rag_llm_r3_66bea0b5", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "rag_llm", "repeat": 3, "order_position": 0, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 6, "evidence_validity_den": 6, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_llm_r3_39601d12", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_llm", "repeat": 3, "order_position": 1, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_scrape_llm_r3_e08f5300", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "scrape_llm", "repeat": 3, "order_position": 2, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 3, "evidence_validity_den": 3, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_rules_r3_3dddd3df", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_rules", "repeat": 3, "order_position": 3, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": false, "verdict_correct": false, "required_checks_passed": false, "evidence_valid_and_supported": false, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": true, "correct_abstention": false, "excessive_abstention": true, "is_resolutive": false, "evidence_validity_num": 0, "evidence_validity_den": 0, "evidence_validity": null, "evidence_support_num": 0, "evidence_support_den": 1, "evidence_support": 0.0, "evidence_coverage_num": 0, "evidence_coverage_den": 2, "evidence_coverage": 0.0, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Selected products [] not in acceptable selections: [['P_THT']]", "Technical verdict 'INSUFFICIENT_EVIDENCE' != gold 'COMPATIBLE'", "Missing required check for requirement 'REQ_R05_PROTOCOL'", "Missing required check for requirement 'REQ_R05_SUPPLY'", "Required evidence group 'group_0' not satisfied by response citations"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "INSUFFICIENT_EVIDENCE", "gold_acceptable": [["P_THT"]], "resp_selected": []}}
{"run_id": "g4_benchmark", "request_id": "req_R05_structured_jev_r3_27c7b9bd", "case_id": "R05", "scenario_family_id": "FAMILY-02-SELECTION", "scenario_id": "S0001", "split": "test", "mode": "M1", "engine": "structured_jev", "repeat": 3, "order_position": 4, "execution_status": "completed", "task_success": false, "content_fully_correct": false, "operationally_valid": true, "execution_completed": true, "selection_correct": true, "verdict_correct": true, "required_checks_passed": false, "evidence_valid_and_supported": true, "commerce_consistent": true, "false_approval": false, "false_rejection": false, "is_abstention": false, "correct_abstention": false, "excessive_abstention": false, "is_resolutive": true, "evidence_validity_num": 12, "evidence_validity_den": 12, "evidence_validity": 1.0, "evidence_support_num": 1, "evidence_support_den": 1, "evidence_support": 1.0, "evidence_coverage_num": 1, "evidence_coverage_den": 2, "evidence_coverage": 0.5, "commerce_num": 0, "commerce_den": 0, "commerce_accuracy": null, "failure_reasons": ["Missing required check for requirement 'REQ_R05_SUPPLY'"], "details": {"selection_applicable": true, "gold_verdict": "COMPATIBLE", "resp_verdict": "COMPATIBLE", "gold_acceptable": [["P_THT"]], "resp_selected": ["P_THT"]}}