fix(evaluation): preserve trace boundaries in eval sets

Author: linhongyu510

tests/test_evaluator.py

@@ -160,3 +160,69 @@ def test_tracing_file_to_evalset():
     )
 
     os.remove(tracing_file_path)
+
+
+def test_tracing_file_creates_isolated_eval_case_per_sorted_trace(tmp_path):
+    trace_a = 101
+    trace_b = 202
+
+    def call_llm(trace_id, start_time, app_name, user_id, prompt="", completion=""):
+        return {
+            "name": "call_llm",
+            "trace_id": trace_id,
+            "start_time": start_time,
+            "attributes": {
+                "gen_ai.app.name": app_name,
+                "gen_ai.user.id": user_id,
+                "gen_ai.prompt.0.content": prompt,
+                "gen_ai.completion.0.content": completion,
+            },
+        }
+
+    def execute_tool(start_time, tool_name):
+        return {
+            "name": f"execute_tool {tool_name}",
+            "trace_id": trace_a,
+            "start_time": start_time,
+            "attributes": {
+                "gen_ai.tool.name": tool_name,
+                "gen_ai.tool.input": json.dumps({"parameters": {"order": tool_name}}),
+                "gen_ai.tool.output": json.dumps({"id": f"call-{tool_name}"}),
+            },
+        }
+
+    tracing_data = [
+        call_llm(trace_a, 40, "app-a", "user-a", completion="answer-a"),
+        call_llm(trace_b, 60, "app-b", "user-b", completion="answer-b"),
+        execute_tool(30, "second"),
+        call_llm(trace_b, 50, "app-b", "user-b", prompt="question-b"),
+        call_llm(trace_a, 10, "app-a", "user-a", prompt="question-a"),
+        execute_tool(20, "first"),
+    ]
+    tracing_file_path = tmp_path / "tracing.json"
+    tracing_file_path.write_text(json.dumps(tracing_data))
+
+    eval_set = BaseEvaluator(
+        agent=None, name="test_evaluator"
+    )._build_eval_set_from_tracing_json(str(tracing_file_path))
+
+    assert len(eval_set.eval_cases) == 2
+    eval_cases = {
+        eval_case.session_input.app_name: eval_case for eval_case in eval_set.eval_cases
+    }
+
+    case_a = eval_cases["app-a"]
+    assert case_a.session_input.user_id == "user-a"
+    assert case_a.creation_timestamp == 10 / 1e9
+    assert case_a.conversation[0].user_content.parts[0].text == "question-a"
+    assert case_a.conversation[0].final_response.parts[0].text == "answer-a"
+    assert [
+        tool.name for tool in case_a.conversation[0].intermediate_data.tool_uses
+    ] == ["first", "second"]
+
+    case_b = eval_cases["app-b"]
+    assert case_b.session_input.user_id == "user-b"
+    assert case_b.creation_timestamp == 50 / 1e9
+    assert case_b.conversation[0].user_content.parts[0].text == "question-b"
+    assert case_b.conversation[0].final_response.parts[0].text == "answer-b"
+    assert case_b.conversation[0].intermediate_data.tool_uses == []

veadk/evaluation/base_evaluator.py

@@ -270,11 +270,11 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet:
                 trace_groups[trace_id] = []
             trace_groups[trace_id].append(span)
 
-        # Convert to evalset format
-        eval_cases, conversation = [], []
-        app_name, user_id = "", ""
-        creation_timestamp = 0
+        # Convert each trace to an isolated eval case.
+        eval_cases = []
         for trace_id, spans in trace_groups.items():
+            spans = sorted(spans, key=lambda span: span["start_time"])
+            conversation = []
             tool_uses = []
 
             # Extract tool_uses from spans with name starting with "execute_tool"
@@ -308,6 +308,9 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet:
             # Extract conversation data from call_llm spans
             user_input = ""
             final_output = ""
+            app_name = ""
+            user_id = ""
+            creation_timestamp = 0
 
             # Find the first call_llm span for user input and the last one for final output
             call_llm_spans = [span for span in spans if span["name"] == "call_llm"]
@@ -347,25 +350,30 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet:
                     }
                 )
 
-        eval_cases.append(
-            {
-                "eval_id": f"veadk_eval_{formatted_timestamp()}",
-                "conversation": conversation,
-                "session_input": {
-                    "app_name": app_name,
-                    "user_id": user_id,
-                    "state": {},
-                },
-                "creation_timestamp": creation_timestamp,
-            }
+                eval_cases.append(
+                    {
+                        "eval_id": (f"veadk_eval_{formatted_timestamp()}_{trace_id}"),
+                        "conversation": conversation,
+                        "session_input": {
+                            "app_name": app_name,
+                            "user_id": user_id,
+                            "state": {},
+                        },
+                        "creation_timestamp": creation_timestamp,
+                    }
+                )
+
+        eval_set_creation_timestamp = min(
+            (eval_case["creation_timestamp"] for eval_case in eval_cases),
+            default=0,
         )
 
         evalset = EvalSet(
             eval_set_id="default",
             name="default",
             description=None,
             eval_cases=eval_cases,
-            creation_timestamp=creation_timestamp,
+            creation_timestamp=eval_set_creation_timestamp,
         )
 
         return evalset