Skip to content

Commit 0d59e6d

Browse files
feat: update eval expected tools for workflow and rule engine prompts
1 parent 7959dff commit 0d59e6d

1 file changed

Lines changed: 10 additions & 3 deletions

File tree

evals/evals.json

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -882,7 +882,9 @@
882882
{
883883
"id": "pack_manifest_entitlement_validation",
884884
"prompt": "I want to load a workflow pack for house production. Tell me what the pack manifest must contain, what entitlement kinds are allowed, and whether a pack can override protected workflow fields like 'endpoint' or 'safety_note'.",
885-
"expected_tools": [],
885+
"expected_tools": [
886+
"kb_search_structured"
887+
],
886888
"expected_resources": [
887889
"fls://capabilities/supported",
888890
"fls://capabilities/not-possible"
@@ -906,7 +908,9 @@
906908
{
907909
"id": "rule_engine_safe_evaluation",
908910
"prompt": "Use the declarative rule engine to find bass tracks with stereo separation above 0.25. Explain which operators are safe, and tell me whether I can pass an '__import__' operator to evaluate arbitrary code.",
909-
"expected_tools": [],
911+
"expected_tools": [
912+
"kb_search_structured"
913+
],
910914
"expected_resources": [
911915
"fls://capabilities/not-possible"
912916
],
@@ -928,7 +932,10 @@
928932
{
929933
"id": "workflow_interaction_request_contract",
930934
"prompt": "Show me how an analysis workflow asks the user to confirm which mixer tracks to include in a low-end analysis, and explain what interaction types are supported.",
931-
"expected_tools": [],
935+
"expected_tools": [
936+
"fl_get_workflow_context",
937+
"fl_review_low_end_stereo"
938+
],
932939
"expected_resources": [
933940
"fls://capabilities/supported"
934941
],

0 commit comments

Comments
 (0)