From 418e91138b483084318781e4ef591e05e9e373c2 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 14 May 2025 16:12:01 +0200 Subject: [PATCH 001/128] Add intersection of track segments with a single line --- .../pandas_geometry_store.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py new file mode 100644 index 000000000..dcd68db07 --- /dev/null +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -0,0 +1,114 @@ +from pandas import DataFrame + + +def create_track_segments(df: DataFrame) -> DataFrame: + """ + Create track segments from a DataFrame containing track points. + + A track segment connects two consecutive points of a track. + + Args: + df (DataFrame): DataFrame with columns 'track-id', 'occurrence', 'x', 'y' + + Returns: + DataFrame: DataFrame with columns 'track-id', 'start-occurrence', 'end-occurrence', + 'start-x', 'start-y', 'end-x', 'end-y' + """ + if df.empty: + return DataFrame() + + # Make a copy to avoid modifying the original DataFrame + data = df.copy() + + # Ensure the DataFrame is sorted by track-id and occurrence + data = data.sort_values(by=['track-id', 'occurrence']) + + # Group by track-id to create segments + grouped = data.groupby('track-id') + + # Create a new DataFrame with shifted values to get start and end points + segments = DataFrame() + segments['track-id'] = data['track-id'] + segments['end-occurrence'] = data['occurrence'] + segments['end-x'] = data['x'] + segments['end-y'] = data['y'] + segments['start-occurrence'] = grouped['occurrence'].shift(1) + segments['start-x'] = grouped['x'].shift(1) + segments['start-y'] = grouped['y'].shift(1) + + # Remove rows where start values are NaN (first point of each track) + segments = segments.dropna() + + # Reset index for clean output + segments = segments.reset_index(drop=True) + + return segments + + +def find_line_intersections(segments_df: DataFrame, line_id: str, start_x: float, start_y: float, + end_x: float, end_y: float) -> DataFrame: + """ + Find intersection points between track segments and a line. + + Args: + segments_df (DataFrame): DataFrame with track segments (output of create_track_segments) + line_id (str): ID of the line + start_x (float): X-coordinate of the line's start point + start_y (float): Y-coordinate of the line's start point + end_x (float): X-coordinate of the line's end point + end_y (float): Y-coordinate of the line's end point + + Returns: + DataFrame: The input DataFrame with additional columns for intersection information: + - 'intersects': Boolean indicating if the segment intersects with the line + - 'intersection_x': X-coordinate of the intersection point (if any) + - 'intersection_y': Y-coordinate of the intersection point (if any) + - 'intersection_line_id': ID of the intersecting line + """ + if segments_df.empty: + return segments_df + + # Create a copy to avoid modifying the original DataFrame + result_df = segments_df.copy() + + # Initialize new columns + result_df['intersects'] = False + result_df['intersection_x'] = None + result_df['intersection_y'] = None + result_df['intersection_line_id'] = None + + # Define line segment coordinates + line_x1, line_y1 = start_x, start_y + line_x2, line_y2 = end_x, end_y + + # Calculate intersection using line segment intersection formula + # First, calculate the denominator for all segments at once + denominator = ((line_y2 - line_y1) * (result_df['end-x'] - result_df['start-x']) - + (line_x2 - line_x1) * (result_df['end-y'] - result_df['start-y'])) + + # Filter out segments where lines are parallel (denominator == 0) + non_parallel_mask = denominator != 0 + + if non_parallel_mask.any(): + # Calculate the parameters for the intersection points + ua = ((line_x2 - line_x1) * (result_df['start-y'] - line_y1) - + (line_y2 - line_y1) * (result_df['start-x'] - line_x1)) / denominator + + ub = ((result_df['end-x'] - result_df['start-x']) * (result_df['start-y'] - line_y1) - + (result_df['end-y'] - result_df['start-y']) * (result_df['start-x'] - line_x1)) / denominator + + # Create a mask for valid intersections (intersection point is on both line segments) + valid_intersection_mask = (0 <= ua) & (ua <= 1) & (0 <= ub) & (ub <= 1) & non_parallel_mask + + if valid_intersection_mask.any(): + # Calculate the intersection points for valid intersections + intersection_x = result_df['start-x'] + ua * (result_df['end-x'] - result_df['start-x']) + intersection_y = result_df['start-y'] + ua * (result_df['end-y'] - result_df['start-y']) + + # Update the result DataFrame with intersection information + result_df.loc[valid_intersection_mask, 'intersects'] = True + result_df.loc[valid_intersection_mask, 'intersection_x'] = intersection_x[valid_intersection_mask] + result_df.loc[valid_intersection_mask, 'intersection_y'] = intersection_y[valid_intersection_mask] + result_df.loc[valid_intersection_mask, 'intersection_line_id'] = line_id + + return result_df \ No newline at end of file From 831208df97d31fef147129884a0c70d82d8b3b07 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 14 May 2025 16:25:55 +0200 Subject: [PATCH 002/128] Add test cases --- .../test_pandas_geometry_store.py | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py new file mode 100644 index 000000000..15e9fcd7c --- /dev/null +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py @@ -0,0 +1,240 @@ +from datetime import datetime + +import pandas as pd +from pandas import DataFrame +from pytest import approx + +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + create_track_segments, + find_line_intersections, +) + + +def test_find_line_intersections_empty_df() -> None: + """Test that an empty DataFrame returns an empty DataFrame.""" + df = DataFrame() + result = find_line_intersections(df, "line1", 0.0, 0.0, 10.0, 10.0) + assert result.empty + + +def test_find_line_intersections_no_intersections() -> None: + """Test with segments that don't intersect with the line.""" + # Create test data + segments_data = { + "track-id": ["track1", "track2"], + "start-x": [10.0, 100.0], + "start-y": [15.0, 150.0], + "end-x": [20.0, 110.0], + "end-y": [25.0, 160.0], + "start-occurrence": [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + ], + "end-occurrence": [ + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + ], + } + segments_df = DataFrame(segments_data) + + # Define a line that doesn't intersect with any segment + line_id = "non_intersecting_line" + start_x = 0.0 + start_y = 100.0 + end_x = 5.0 + end_y = 105.0 + + # Find intersections + result = find_line_intersections( + segments_df, line_id, start_x, start_y, end_x, end_y + ) + + # Check that no segments intersect + assert not result["intersects"].any() + assert (result["intersection_x"].isna()).all() + assert (result["intersection_y"].isna()).all() + assert (result["intersection_line_id"].isna()).all() + + +def test_find_line_intersections_with_intersections() -> None: + """Test with segments that intersect with the line.""" + # Create test data + segments_data = { + "track-id": ["track1", "track2"], + "start-x": [10.0, 100.0], + "start-y": [10.0, 150.0], + "end-x": [20.0, 110.0], + "end-y": [20.0, 160.0], + "start-occurrence": [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + ], + "end-occurrence": [ + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + ], + } + segments_df = DataFrame(segments_data) + + # Define a line that intersects with the first segment + line_id = "intersecting_line" + start_x = 0.0 + start_y = 0.0 + end_x = 30.0 + end_y = 30.0 + + # Find intersections + result = find_line_intersections( + segments_df, line_id, start_x, start_y, end_x, end_y + ) + + # Check that the first segment intersects + assert result.iloc[0]["intersects"] + assert result.iloc[0]["intersection_x"] == approx(15.0) + assert result.iloc[0]["intersection_y"] == approx(15.0) + assert result.iloc[0]["intersection_line_id"] == line_id + + # Check that the second segment doesn't intersect + assert not result.iloc[1]["intersects"] + assert pd.isna(result.iloc[1]["intersection_x"]) + assert pd.isna(result.iloc[1]["intersection_y"]) + assert pd.isna(result.iloc[1]["intersection_line_id"]) + + +def test_find_line_intersections_multiple_intersections() -> None: + """Test with multiple segments intersecting with the line.""" + # Create test data + segments_data = { + "track-id": ["track1", "track2", "track3"], + "start-x": [10.0, 100.0, 30.0], + "start-y": [10.0, 10.0, 30.0], + "end-x": [20.0, 110.0, 40.0], + "end-y": [20.0, 20.0, 40.0], + "start-occurrence": [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + ], + "end-occurrence": [ + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + ], + } + segments_df = DataFrame(segments_data) + + # Define a line that intersects with the first and third segments + line_id = "multiple_intersecting_line" + start_x = 0.0 + start_y = 0.0 + end_x = 50.0 + end_y = 50.0 + + # Find intersections + result = find_line_intersections( + segments_df, line_id, start_x, start_y, end_x, end_y + ) + + # Check that the first and third segments intersect + assert result.iloc[0]["intersects"] + assert result.iloc[0]["intersection_x"] == approx(15.0) + assert result.iloc[0]["intersection_y"] == approx(15.0) + assert result.iloc[0]["intersection_line_id"] == line_id + + assert not result.iloc[1]["intersects"] + + assert result.iloc[2]["intersects"] + assert result.iloc[2]["intersection_x"] == approx(35.0) + assert result.iloc[2]["intersection_y"] == approx(35.0) + assert result.iloc[2]["intersection_line_id"] == line_id + + # Check the count of intersecting segments + assert result["intersects"].sum() == 2 + + +def test_create_track_segments_empty_df() -> None: + """Test that an empty DataFrame returns an empty DataFrame.""" + df = DataFrame() + result = create_track_segments(df) + assert result.empty + + +def test_create_track_segments_single_track() -> None: + """Test creating segments from a single track with multiple points.""" + # Create test data + data = { + "track-id": ["track1", "track1", "track1"], + "occurrence": [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 2), + ], + "x": [10.0, 20.0, 30.0], + "y": [15.0, 25.0, 35.0], + } + df = DataFrame(data) + + # Create segments + result = create_track_segments(df) + + # Check result + assert len(result) == 2 # Should have 2 segments for 3 points + + # Check first segment + assert result.iloc[0]["track-id"] == "track1" + assert result.iloc[0]["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 0) + assert result.iloc[0]["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) + assert result.iloc[0]["start-x"] == 10.0 + assert result.iloc[0]["start-y"] == 15.0 + assert result.iloc[0]["end-x"] == 20.0 + assert result.iloc[0]["end-y"] == 25.0 + + # Check second segment + assert result.iloc[1]["track-id"] == "track1" + assert result.iloc[1]["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) + assert result.iloc[1]["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 2) + assert result.iloc[1]["start-x"] == 20.0 + assert result.iloc[1]["start-y"] == 25.0 + assert result.iloc[1]["end-x"] == 30.0 + assert result.iloc[1]["end-y"] == 35.0 + + +def test_create_track_segments_multiple_tracks() -> None: + """Test creating segments from multiple tracks.""" + # Create test data + data = { + "track-id": ["track1", "track1", "track2", "track2"], + "occurrence": [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + ], + "x": [10.0, 20.0, 100.0, 110.0], + "y": [15.0, 25.0, 150.0, 160.0], + } + df = DataFrame(data) + + # Create segments + result = create_track_segments(df) + + # Check result + assert len(result) == 2 # Should have 2 segments (1 for each track) + + # Check track1 segment + track1_segment = result[result["track-id"] == "track1"].iloc[0] + assert track1_segment["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 0) + assert track1_segment["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) + assert track1_segment["start-x"] == 10.0 + assert track1_segment["start-y"] == 15.0 + assert track1_segment["end-x"] == 20.0 + assert track1_segment["end-y"] == 25.0 + + # Check track2 segment + track2_segment = result[result["track-id"] == "track2"].iloc[0] + assert track2_segment["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 0) + assert track2_segment["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) + assert track2_segment["start-x"] == 100.0 + assert track2_segment["start-y"] == 150.0 + assert track2_segment["end-x"] == 110.0 + assert track2_segment["end-y"] == 160.0 From 854b7fd555a814e104f067d44762e2e17df4226d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 15 May 2025 11:55:54 +0200 Subject: [PATCH 003/128] Introduce constants --- .../pandas_geometry_store.py | 129 ++++++++++++------ 1 file changed, 87 insertions(+), 42 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index dcd68db07..e237b8cf4 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -1,5 +1,25 @@ from pandas import DataFrame +# Column names for track points +TRACK_ID = "track-id" +OCCURRENCE = "occurrence" +X = "x" +Y = "y" + +# Column names for track segments +START_OCCURRENCE = "start-occurrence" +END_OCCURRENCE = "end-occurrence" +START_X = "start-x" +START_Y = "start-y" +END_X = "end-x" +END_Y = "end-y" + +# Column names for intersections +INTERSECTS = "intersects" +INTERSECTION_X = "intersection_x" +INTERSECTION_Y = "intersection_y" +INTERSECTION_LINE_ID = "intersection_line_id" + def create_track_segments(df: DataFrame) -> DataFrame: """ @@ -8,11 +28,11 @@ def create_track_segments(df: DataFrame) -> DataFrame: A track segment connects two consecutive points of a track. Args: - df (DataFrame): DataFrame with columns 'track-id', 'occurrence', 'x', 'y' + df (DataFrame): DataFrame with columns TRACK_ID, OCCURRENCE, X, Y Returns: - DataFrame: DataFrame with columns 'track-id', 'start-occurrence', 'end-occurrence', - 'start-x', 'start-y', 'end-x', 'end-y' + DataFrame: DataFrame with columns TRACK_ID, START_OCCURRENCE, END_OCCURRENCE, + START_X, START_Y, END_X, END_Y """ if df.empty: return DataFrame() @@ -21,20 +41,20 @@ def create_track_segments(df: DataFrame) -> DataFrame: data = df.copy() # Ensure the DataFrame is sorted by track-id and occurrence - data = data.sort_values(by=['track-id', 'occurrence']) + data = data.sort_values(by=[TRACK_ID, OCCURRENCE]) # Group by track-id to create segments - grouped = data.groupby('track-id') + grouped = data.groupby(TRACK_ID) # Create a new DataFrame with shifted values to get start and end points segments = DataFrame() - segments['track-id'] = data['track-id'] - segments['end-occurrence'] = data['occurrence'] - segments['end-x'] = data['x'] - segments['end-y'] = data['y'] - segments['start-occurrence'] = grouped['occurrence'].shift(1) - segments['start-x'] = grouped['x'].shift(1) - segments['start-y'] = grouped['y'].shift(1) + segments[TRACK_ID] = data[TRACK_ID] + segments[END_OCCURRENCE] = data[OCCURRENCE] + segments[END_X] = data[X] + segments[END_Y] = data[Y] + segments[START_OCCURRENCE] = grouped[OCCURRENCE].shift(1) + segments[START_X] = grouped[X].shift(1) + segments[START_Y] = grouped[Y].shift(1) # Remove rows where start values are NaN (first point of each track) segments = segments.dropna() @@ -45,13 +65,20 @@ def create_track_segments(df: DataFrame) -> DataFrame: return segments -def find_line_intersections(segments_df: DataFrame, line_id: str, start_x: float, start_y: float, - end_x: float, end_y: float) -> DataFrame: +def find_line_intersections( + segments_df: DataFrame, + line_id: str, + start_x: float, + start_y: float, + end_x: float, + end_y: float, +) -> DataFrame: """ Find intersection points between track segments and a line. Args: - segments_df (DataFrame): DataFrame with track segments (output of create_track_segments) + segments_df (DataFrame): DataFrame with track segments (output of + create_track_segments) line_id (str): ID of the line start_x (float): X-coordinate of the line's start point start_y (float): Y-coordinate of the line's start point @@ -59,11 +86,13 @@ def find_line_intersections(segments_df: DataFrame, line_id: str, start_x: float end_y (float): Y-coordinate of the line's end point Returns: - DataFrame: The input DataFrame with additional columns for intersection information: - - 'intersects': Boolean indicating if the segment intersects with the line - - 'intersection_x': X-coordinate of the intersection point (if any) - - 'intersection_y': Y-coordinate of the intersection point (if any) - - 'intersection_line_id': ID of the intersecting line + DataFrame: The input DataFrame with additional columns for intersection + information: + - INTERSECTS: Boolean indicating if the segment intersects with the + line + - INTERSECTION_X: X-coordinate of the intersection point (if any) + - INTERSECTION_Y: Y-coordinate of the intersection point (if any) + - INTERSECTION_LINE_ID: ID of the intersecting line """ if segments_df.empty: return segments_df @@ -72,10 +101,10 @@ def find_line_intersections(segments_df: DataFrame, line_id: str, start_x: float result_df = segments_df.copy() # Initialize new columns - result_df['intersects'] = False - result_df['intersection_x'] = None - result_df['intersection_y'] = None - result_df['intersection_line_id'] = None + result_df[INTERSECTS] = False + result_df[INTERSECTION_X] = None + result_df[INTERSECTION_Y] = None + result_df[INTERSECTION_LINE_ID] = None # Define line segment coordinates line_x1, line_y1 = start_x, start_y @@ -83,32 +112,48 @@ def find_line_intersections(segments_df: DataFrame, line_id: str, start_x: float # Calculate intersection using line segment intersection formula # First, calculate the denominator for all segments at once - denominator = ((line_y2 - line_y1) * (result_df['end-x'] - result_df['start-x']) - - (line_x2 - line_x1) * (result_df['end-y'] - result_df['start-y'])) + denominator = (line_y2 - line_y1) * (result_df[END_X] - result_df[START_X]) - ( + line_x2 - line_x1 + ) * (result_df[END_Y] - result_df[START_Y]) # Filter out segments where lines are parallel (denominator == 0) non_parallel_mask = denominator != 0 if non_parallel_mask.any(): # Calculate the parameters for the intersection points - ua = ((line_x2 - line_x1) * (result_df['start-y'] - line_y1) - - (line_y2 - line_y1) * (result_df['start-x'] - line_x1)) / denominator - - ub = ((result_df['end-x'] - result_df['start-x']) * (result_df['start-y'] - line_y1) - - (result_df['end-y'] - result_df['start-y']) * (result_df['start-x'] - line_x1)) / denominator - - # Create a mask for valid intersections (intersection point is on both line segments) - valid_intersection_mask = (0 <= ua) & (ua <= 1) & (0 <= ub) & (ub <= 1) & non_parallel_mask + ua = ( + (line_x2 - line_x1) * (result_df[START_Y] - line_y1) + - (line_y2 - line_y1) * (result_df[START_X] - line_x1) + ) / denominator + + ub = ( + (result_df[END_X] - result_df[START_X]) * (result_df[START_Y] - line_y1) + - (result_df[END_Y] - result_df[START_Y]) * (result_df[START_X] - line_x1) + ) / denominator + + # Create a mask for valid intersections (intersection point is on both line + # segments) + valid_intersection_mask = ( + (0 <= ua) & (ua <= 1) & (0 <= ub) & (ub <= 1) & non_parallel_mask + ) if valid_intersection_mask.any(): # Calculate the intersection points for valid intersections - intersection_x = result_df['start-x'] + ua * (result_df['end-x'] - result_df['start-x']) - intersection_y = result_df['start-y'] + ua * (result_df['end-y'] - result_df['start-y']) + intersection_x = result_df[START_X] + ua * ( + result_df[END_X] - result_df[START_X] + ) + intersection_y = result_df[START_Y] + ua * ( + result_df[END_Y] - result_df[START_Y] + ) # Update the result DataFrame with intersection information - result_df.loc[valid_intersection_mask, 'intersects'] = True - result_df.loc[valid_intersection_mask, 'intersection_x'] = intersection_x[valid_intersection_mask] - result_df.loc[valid_intersection_mask, 'intersection_y'] = intersection_y[valid_intersection_mask] - result_df.loc[valid_intersection_mask, 'intersection_line_id'] = line_id - - return result_df \ No newline at end of file + result_df.loc[valid_intersection_mask, INTERSECTS] = True + result_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_x[ + valid_intersection_mask + ] + result_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_y[ + valid_intersection_mask + ] + result_df.loc[valid_intersection_mask, INTERSECTION_LINE_ID] = line_id + + return result_df From 13915c058e6c70528f3d44cb0b3902ed8e2519c0 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 15 May 2025 12:02:34 +0200 Subject: [PATCH 004/128] Use constants in tests --- .../test_pandas_geometry_store.py | 172 ++++++++++-------- 1 file changed, 93 insertions(+), 79 deletions(-) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py index 15e9fcd7c..2265e2748 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py @@ -5,6 +5,20 @@ from pytest import approx from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + END_OCCURRENCE, + END_X, + END_Y, + INTERSECTION_LINE_ID, + INTERSECTION_X, + INTERSECTION_Y, + INTERSECTS, + OCCURRENCE, + START_OCCURRENCE, + START_X, + START_Y, + TRACK_ID, + X, + Y, create_track_segments, find_line_intersections, ) @@ -21,16 +35,16 @@ def test_find_line_intersections_no_intersections() -> None: """Test with segments that don't intersect with the line.""" # Create test data segments_data = { - "track-id": ["track1", "track2"], - "start-x": [10.0, 100.0], - "start-y": [15.0, 150.0], - "end-x": [20.0, 110.0], - "end-y": [25.0, 160.0], - "start-occurrence": [ + TRACK_ID: ["track1", "track2"], + START_X: [10.0, 100.0], + START_Y: [15.0, 150.0], + END_X: [20.0, 110.0], + END_Y: [25.0, 160.0], + START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), ], - "end-occurrence": [ + END_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 1), ], @@ -50,26 +64,26 @@ def test_find_line_intersections_no_intersections() -> None: ) # Check that no segments intersect - assert not result["intersects"].any() - assert (result["intersection_x"].isna()).all() - assert (result["intersection_y"].isna()).all() - assert (result["intersection_line_id"].isna()).all() + assert not result[INTERSECTS].any() + assert (result[INTERSECTION_X].isna()).all() + assert (result[INTERSECTION_Y].isna()).all() + assert (result[INTERSECTION_LINE_ID].isna()).all() def test_find_line_intersections_with_intersections() -> None: """Test with segments that intersect with the line.""" # Create test data segments_data = { - "track-id": ["track1", "track2"], - "start-x": [10.0, 100.0], - "start-y": [10.0, 150.0], - "end-x": [20.0, 110.0], - "end-y": [20.0, 160.0], - "start-occurrence": [ + TRACK_ID: ["track1", "track2"], + START_X: [10.0, 100.0], + START_Y: [10.0, 150.0], + END_X: [20.0, 110.0], + END_Y: [20.0, 160.0], + START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), ], - "end-occurrence": [ + END_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 1), ], @@ -89,33 +103,33 @@ def test_find_line_intersections_with_intersections() -> None: ) # Check that the first segment intersects - assert result.iloc[0]["intersects"] - assert result.iloc[0]["intersection_x"] == approx(15.0) - assert result.iloc[0]["intersection_y"] == approx(15.0) - assert result.iloc[0]["intersection_line_id"] == line_id + assert result.iloc[0][INTERSECTS] + assert result.iloc[0][INTERSECTION_X] == approx(15.0) + assert result.iloc[0][INTERSECTION_Y] == approx(15.0) + assert result.iloc[0][INTERSECTION_LINE_ID] == line_id # Check that the second segment doesn't intersect - assert not result.iloc[1]["intersects"] - assert pd.isna(result.iloc[1]["intersection_x"]) - assert pd.isna(result.iloc[1]["intersection_y"]) - assert pd.isna(result.iloc[1]["intersection_line_id"]) + assert not result.iloc[1][INTERSECTS] + assert pd.isna(result.iloc[1][INTERSECTION_X]) + assert pd.isna(result.iloc[1][INTERSECTION_Y]) + assert pd.isna(result.iloc[1][INTERSECTION_LINE_ID]) def test_find_line_intersections_multiple_intersections() -> None: """Test with multiple segments intersecting with the line.""" # Create test data segments_data = { - "track-id": ["track1", "track2", "track3"], - "start-x": [10.0, 100.0, 30.0], - "start-y": [10.0, 10.0, 30.0], - "end-x": [20.0, 110.0, 40.0], - "end-y": [20.0, 20.0, 40.0], - "start-occurrence": [ + TRACK_ID: ["track1", "track2", "track3"], + START_X: [10.0, 100.0, 30.0], + START_Y: [10.0, 10.0, 30.0], + END_X: [20.0, 110.0, 40.0], + END_Y: [20.0, 20.0, 40.0], + START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), ], - "end-occurrence": [ + END_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 1), @@ -136,20 +150,20 @@ def test_find_line_intersections_multiple_intersections() -> None: ) # Check that the first and third segments intersect - assert result.iloc[0]["intersects"] - assert result.iloc[0]["intersection_x"] == approx(15.0) - assert result.iloc[0]["intersection_y"] == approx(15.0) - assert result.iloc[0]["intersection_line_id"] == line_id + assert result.iloc[0][INTERSECTS] + assert result.iloc[0][INTERSECTION_X] == approx(15.0) + assert result.iloc[0][INTERSECTION_Y] == approx(15.0) + assert result.iloc[0][INTERSECTION_LINE_ID] == line_id - assert not result.iloc[1]["intersects"] + assert not result.iloc[1][INTERSECTS] - assert result.iloc[2]["intersects"] - assert result.iloc[2]["intersection_x"] == approx(35.0) - assert result.iloc[2]["intersection_y"] == approx(35.0) - assert result.iloc[2]["intersection_line_id"] == line_id + assert result.iloc[2][INTERSECTS] + assert result.iloc[2][INTERSECTION_X] == approx(35.0) + assert result.iloc[2][INTERSECTION_Y] == approx(35.0) + assert result.iloc[2][INTERSECTION_LINE_ID] == line_id # Check the count of intersecting segments - assert result["intersects"].sum() == 2 + assert result[INTERSECTS].sum() == 2 def test_create_track_segments_empty_df() -> None: @@ -163,14 +177,14 @@ def test_create_track_segments_single_track() -> None: """Test creating segments from a single track with multiple points.""" # Create test data data = { - "track-id": ["track1", "track1", "track1"], - "occurrence": [ + TRACK_ID: ["track1", "track1", "track1"], + OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 2), ], - "x": [10.0, 20.0, 30.0], - "y": [15.0, 25.0, 35.0], + X: [10.0, 20.0, 30.0], + Y: [15.0, 25.0, 35.0], } df = DataFrame(data) @@ -181,37 +195,37 @@ def test_create_track_segments_single_track() -> None: assert len(result) == 2 # Should have 2 segments for 3 points # Check first segment - assert result.iloc[0]["track-id"] == "track1" - assert result.iloc[0]["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 0) - assert result.iloc[0]["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) - assert result.iloc[0]["start-x"] == 10.0 - assert result.iloc[0]["start-y"] == 15.0 - assert result.iloc[0]["end-x"] == 20.0 - assert result.iloc[0]["end-y"] == 25.0 + assert result.iloc[0][TRACK_ID] == "track1" + assert result.iloc[0][START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 0) + assert result.iloc[0][END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) + assert result.iloc[0][START_X] == 10.0 + assert result.iloc[0][START_Y] == 15.0 + assert result.iloc[0][END_X] == 20.0 + assert result.iloc[0][END_Y] == 25.0 # Check second segment - assert result.iloc[1]["track-id"] == "track1" - assert result.iloc[1]["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) - assert result.iloc[1]["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 2) - assert result.iloc[1]["start-x"] == 20.0 - assert result.iloc[1]["start-y"] == 25.0 - assert result.iloc[1]["end-x"] == 30.0 - assert result.iloc[1]["end-y"] == 35.0 + assert result.iloc[1][TRACK_ID] == "track1" + assert result.iloc[1][START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) + assert result.iloc[1][END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 2) + assert result.iloc[1][START_X] == 20.0 + assert result.iloc[1][START_Y] == 25.0 + assert result.iloc[1][END_X] == 30.0 + assert result.iloc[1][END_Y] == 35.0 def test_create_track_segments_multiple_tracks() -> None: """Test creating segments from multiple tracks.""" # Create test data data = { - "track-id": ["track1", "track1", "track2", "track2"], - "occurrence": [ + TRACK_ID: ["track1", "track1", "track2", "track2"], + OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 1), ], - "x": [10.0, 20.0, 100.0, 110.0], - "y": [15.0, 25.0, 150.0, 160.0], + X: [10.0, 20.0, 100.0, 110.0], + Y: [15.0, 25.0, 150.0, 160.0], } df = DataFrame(data) @@ -222,19 +236,19 @@ def test_create_track_segments_multiple_tracks() -> None: assert len(result) == 2 # Should have 2 segments (1 for each track) # Check track1 segment - track1_segment = result[result["track-id"] == "track1"].iloc[0] - assert track1_segment["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 0) - assert track1_segment["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) - assert track1_segment["start-x"] == 10.0 - assert track1_segment["start-y"] == 15.0 - assert track1_segment["end-x"] == 20.0 - assert track1_segment["end-y"] == 25.0 + track1_segment = result[result[TRACK_ID] == "track1"].iloc[0] + assert track1_segment[START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 0) + assert track1_segment[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) + assert track1_segment[START_X] == 10.0 + assert track1_segment[START_Y] == 15.0 + assert track1_segment[END_X] == 20.0 + assert track1_segment[END_Y] == 25.0 # Check track2 segment - track2_segment = result[result["track-id"] == "track2"].iloc[0] - assert track2_segment["start-occurrence"] == datetime(2023, 1, 1, 10, 0, 0) - assert track2_segment["end-occurrence"] == datetime(2023, 1, 1, 10, 0, 1) - assert track2_segment["start-x"] == 100.0 - assert track2_segment["start-y"] == 150.0 - assert track2_segment["end-x"] == 110.0 - assert track2_segment["end-y"] == 160.0 + track2_segment = result[result[TRACK_ID] == "track2"].iloc[0] + assert track2_segment[START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 0) + assert track2_segment[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) + assert track2_segment[START_X] == 100.0 + assert track2_segment[START_Y] == 150.0 + assert track2_segment[END_X] == 110.0 + assert track2_segment[END_Y] == 160.0 From 016be457e323310ce60cdfde9c0729b1cfa10c6c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 15 May 2025 14:44:11 +0200 Subject: [PATCH 005/128] Separate functionality --- .../pandas_geometry_store.py | 255 ++++++++++++++---- 1 file changed, 209 insertions(+), 46 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index e237b8cf4..0627b6aa9 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -20,6 +20,12 @@ INTERSECTION_Y = "intersection_y" INTERSECTION_LINE_ID = "intersection_line_id" +# Column names for intersection parameters +DENOMINATOR = "denominator" +NON_PARALLEL = "non_parallel" +UA = "ua" +UB = "ub" + def create_track_segments(df: DataFrame) -> DataFrame: """ @@ -65,6 +71,192 @@ def create_track_segments(df: DataFrame) -> DataFrame: return segments +def calculate_intersection_parameters( + segments_df: DataFrame, + line_x1: float, + line_y1: float, + line_x2: float, + line_y2: float, +) -> DataFrame: + """ + Calculate parameters needed for line intersection calculations. + + Args: + segments_df (DataFrame): DataFrame with track segments + line_x1 (float): X-coordinate of the line's start point + line_y1 (float): Y-coordinate of the line's start point + line_x2 (float): X-coordinate of the line's end point + line_y2 (float): Y-coordinate of the line's end point + + Returns: + DataFrame: A DataFrame containing the following columns: + - DENOMINATOR: Denominator values for intersection calculations + - NON_PARALLEL: Boolean mask for non-parallel segments + - UA: Parameter ua for intersection point calculation + - UB: Parameter ub for intersection point calculation + """ + # Initialize result DataFrame + result_df = DataFrame(index=segments_df.index) + + # Calculate denominator for all segments at once + result_df[DENOMINATOR] = (line_y2 - line_y1) * ( + segments_df[END_X] - segments_df[START_X] + ) - (line_x2 - line_x1) * (segments_df[END_Y] - segments_df[START_Y]) + + # Filter out segments where lines are parallel (denominator == 0) + result_df[NON_PARALLEL] = result_df[DENOMINATOR] != 0 + + # Initialize ua and ub columns + result_df[UA] = None + result_df[UB] = None + + if result_df[NON_PARALLEL].any(): + # Calculate the parameters for the intersection points + ua_values = ( + (line_x2 - line_x1) * (segments_df[START_Y] - line_y1) + - (line_y2 - line_y1) * (segments_df[START_X] - line_x1) + ) / result_df[DENOMINATOR] + + ub_values = ( + (segments_df[END_X] - segments_df[START_X]) + * (segments_df[START_Y] - line_y1) + - (segments_df[END_Y] - segments_df[START_Y]) + * (segments_df[START_X] - line_x1) + ) / result_df[DENOMINATOR] + + # Update ua and ub columns for non-parallel segments + result_df.loc[result_df[NON_PARALLEL], UA] = ua_values[result_df[NON_PARALLEL]] + result_df.loc[result_df[NON_PARALLEL], UB] = ub_values[result_df[NON_PARALLEL]] + + return result_df + + +def check_line_intersections( + segments_df: DataFrame, + line_x1: float, + line_y1: float, + line_x2: float, + line_y2: float, +) -> DataFrame: + """ + Check if track segments intersect with a line. + + Args: + segments_df (DataFrame): DataFrame with track segments + line_x1 (float): X-coordinate of the line's start point + line_y1 (float): Y-coordinate of the line's start point + line_x2 (float): X-coordinate of the line's end point + line_y2 (float): Y-coordinate of the line's end point + + Returns: + DataFrame: Boolean mask indicating which segments intersect with the line + """ + if segments_df.empty: + return segments_df + + # Calculate intersection parameters + params_df = calculate_intersection_parameters( + segments_df, line_x1, line_y1, line_x2, line_y2 + ) + + # If all segments are parallel or ua/ub couldn't be calculated + if ( + not params_df[NON_PARALLEL].any() + or params_df[UA].isna().all() + or params_df[UB].isna().all() + ): + return DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) + + # Create a mask for valid intersections (intersection point is on both line + # segments) + valid_intersection_mask = ( + (0 <= params_df[UA]) + & (params_df[UA] <= 1) + & (0 <= params_df[UB]) + & (params_df[UB] <= 1) + & params_df[NON_PARALLEL] + ) + + # Create a DataFrame with the intersection results + intersects_df = DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) + intersects_df.loc[valid_intersection_mask, INTERSECTS] = True + + return intersects_df + + +def calculate_intersection_points( + segments_df: DataFrame, + line_x1: float, + line_y1: float, + line_x2: float, + line_y2: float, +) -> DataFrame: + """ + Calculate intersection points between track segments and a line. + + Args: + segments_df (DataFrame): DataFrame with track segments + line_x1 (float): X-coordinate of the line's start point + line_y1 (float): Y-coordinate of the line's start point + line_x2 (float): X-coordinate of the line's end point + line_y2 (float): Y-coordinate of the line's end point + + Returns: + DataFrame: DataFrame with columns INTERSECTION_X and INTERSECTION_Y containing + the coordinates of intersection points + """ + if segments_df.empty: + return DataFrame( + index=segments_df.index, columns=[INTERSECTION_X, INTERSECTION_Y] + ) + + # Calculate intersection parameters + params_df = calculate_intersection_parameters( + segments_df, line_x1, line_y1, line_x2, line_y2 + ) + + # Initialize DataFrame for intersection coordinates + intersection_df = DataFrame( + index=segments_df.index, columns=[INTERSECTION_X, INTERSECTION_Y] + ) + + # If all segments are parallel or ua/ub couldn't be calculated + if ( + not params_df[NON_PARALLEL].any() + or params_df[UA].isna().all() + or params_df[UB].isna().all() + ): + return intersection_df + + # Create a mask for valid intersections + valid_intersection_mask = ( + (0 <= params_df[UA]) + & (params_df[UA] <= 1) + & (0 <= params_df[UB]) + & (params_df[UB] <= 1) + & params_df[NON_PARALLEL] + ) + + if valid_intersection_mask.any(): + # Calculate the intersection points for valid intersections + intersection_x = segments_df[START_X] + params_df[UA] * ( + segments_df[END_X] - segments_df[START_X] + ) + intersection_y = segments_df[START_Y] + params_df[UA] * ( + segments_df[END_Y] - segments_df[START_Y] + ) + + # Update the result DataFrame with intersection coordinates + intersection_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_x[ + valid_intersection_mask + ] + intersection_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_y[ + valid_intersection_mask + ] + + return intersection_df + + def find_line_intersections( segments_df: DataFrame, line_id: str, @@ -106,54 +298,25 @@ def find_line_intersections( result_df[INTERSECTION_Y] = None result_df[INTERSECTION_LINE_ID] = None - # Define line segment coordinates - line_x1, line_y1 = start_x, start_y - line_x2, line_y2 = end_x, end_y - - # Calculate intersection using line segment intersection formula - # First, calculate the denominator for all segments at once - denominator = (line_y2 - line_y1) * (result_df[END_X] - result_df[START_X]) - ( - line_x2 - line_x1 - ) * (result_df[END_Y] - result_df[START_Y]) + # Check which segments intersect with the line + intersects_df = check_line_intersections(result_df, start_x, start_y, end_x, end_y) - # Filter out segments where lines are parallel (denominator == 0) - non_parallel_mask = denominator != 0 - - if non_parallel_mask.any(): - # Calculate the parameters for the intersection points - ua = ( - (line_x2 - line_x1) * (result_df[START_Y] - line_y1) - - (line_y2 - line_y1) * (result_df[START_X] - line_x1) - ) / denominator - - ub = ( - (result_df[END_X] - result_df[START_X]) * (result_df[START_Y] - line_y1) - - (result_df[END_Y] - result_df[START_Y]) * (result_df[START_X] - line_x1) - ) / denominator - - # Create a mask for valid intersections (intersection point is on both line - # segments) - valid_intersection_mask = ( - (0 <= ua) & (ua <= 1) & (0 <= ub) & (ub <= 1) & non_parallel_mask + # If there are any intersections + if intersects_df[INTERSECTS].any(): + # Calculate intersection points + intersection_df = calculate_intersection_points( + result_df, start_x, start_y, end_x, end_y ) - if valid_intersection_mask.any(): - # Calculate the intersection points for valid intersections - intersection_x = result_df[START_X] + ua * ( - result_df[END_X] - result_df[START_X] - ) - intersection_y = result_df[START_Y] + ua * ( - result_df[END_Y] - result_df[START_Y] - ) - - # Update the result DataFrame with intersection information - result_df.loc[valid_intersection_mask, INTERSECTS] = True - result_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_x[ - valid_intersection_mask - ] - result_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_y[ - valid_intersection_mask - ] - result_df.loc[valid_intersection_mask, INTERSECTION_LINE_ID] = line_id + # Update the result DataFrame with intersection information + valid_intersection_mask = intersects_df[INTERSECTS] + result_df.loc[valid_intersection_mask, INTERSECTS] = True + result_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_df.loc[ + valid_intersection_mask, INTERSECTION_X + ] + result_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_df.loc[ + valid_intersection_mask, INTERSECTION_Y + ] + result_df.loc[valid_intersection_mask, INTERSECTION_LINE_ID] = line_id return result_df From d7661521385ca84e9f9d421eb10d1327424e6db2 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 15 May 2025 21:00:05 +0200 Subject: [PATCH 006/128] Intersect a polygon --- .../pandas_geometry_store.py | 53 ++++++++ .../test_pandas_geometry_store.py | 117 ++++++++++++++++-- 2 files changed, 162 insertions(+), 8 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 0627b6aa9..6f3742957 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -1,5 +1,7 @@ from pandas import DataFrame +from OTAnalytics.domain.geometry import Polygon + # Column names for track points TRACK_ID = "track-id" OCCURRENCE = "occurrence" @@ -320,3 +322,54 @@ def find_line_intersections( result_df.loc[valid_intersection_mask, INTERSECTION_LINE_ID] = line_id return result_df + + +def check_polygon_intersections( + segments_df: DataFrame, + polygon: Polygon, +) -> DataFrame: + """ + Check if track segments intersect with a polygon. + + A polygon consists of a set of lines. This function checks if each track segment + intersects with any of the lines in the polygon. + + Args: + segments_df (DataFrame): DataFrame with track segments (output of + create_track_segments) + polygon (Polygon): The polygon to check intersections with + + Returns: + DataFrame: The input DataFrame with an additional column "intersects-polygon" + indicating if the segment intersects with the polygon + """ + if segments_df.empty: + return segments_df + + # Create a copy to avoid modifying the original DataFrame + result_df = segments_df.copy() + + # Initialize the intersects-polygon column to False + result_df["intersects-polygon"] = False + + # Get the coordinates of the polygon + polygon_coordinates = polygon.coordinates + + # Check intersections with each line segment of the polygon + for i in range(len(polygon_coordinates) - 1): + # Get the start and end points of the current line segment + start_x = polygon_coordinates[i].x + start_y = polygon_coordinates[i].y + end_x = polygon_coordinates[i + 1].x + end_y = polygon_coordinates[i + 1].y + + # Check which segments intersect with the current line segment + intersects_df = check_line_intersections( + result_df, start_x, start_y, end_x, end_y + ) + + # Update the result DataFrame + if intersects_df[INTERSECTS].any(): + result_df.loc[intersects_df[INTERSECTS], "intersects-polygon"] = True + + return result_df diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py index 2265e2748..fc8270217 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py @@ -4,6 +4,7 @@ from pandas import DataFrame from pytest import approx +from OTAnalytics.domain.geometry import Coordinate, Polygon from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( END_OCCURRENCE, END_X, @@ -19,6 +20,7 @@ TRACK_ID, X, Y, + check_polygon_intersections, create_track_segments, find_line_intersections, ) @@ -75,10 +77,10 @@ def test_find_line_intersections_with_intersections() -> None: # Create test data segments_data = { TRACK_ID: ["track1", "track2"], - START_X: [10.0, 100.0], - START_Y: [10.0, 150.0], - END_X: [20.0, 110.0], - END_Y: [20.0, 160.0], + START_X: [15.0, 100.0], + START_Y: [0.0, 150.0], + END_X: [15.0, 110.0], + END_Y: [30.0, 160.0], START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), @@ -120,10 +122,10 @@ def test_find_line_intersections_multiple_intersections() -> None: # Create test data segments_data = { TRACK_ID: ["track1", "track2", "track3"], - START_X: [10.0, 100.0, 30.0], - START_Y: [10.0, 10.0, 30.0], - END_X: [20.0, 110.0, 40.0], - END_Y: [20.0, 20.0, 40.0], + START_X: [20.0, 100.0, 30.0], + START_Y: [10.0, 10.0, 40.0], + END_X: [10.0, 110.0, 40.0], + END_Y: [20.0, 20.0, 30.0], START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), @@ -252,3 +254,102 @@ def test_create_track_segments_multiple_tracks() -> None: assert track2_segment[START_Y] == 150.0 assert track2_segment[END_X] == 110.0 assert track2_segment[END_Y] == 160.0 + + +def test_check_polygon_intersections_empty_df() -> None: + """Test that an empty DataFrame returns an empty DataFrame.""" + df = DataFrame() + polygon = Polygon( + [ + Coordinate(0.0, 0.0), + Coordinate(10.0, 0.0), + Coordinate(10.0, 10.0), + Coordinate(0.0, 10.0), + Coordinate(0.0, 0.0), + ] + ) + result = check_polygon_intersections(df, polygon) + assert result.empty + + +def test_check_polygon_intersections_no_intersections() -> None: + """Test with segments that don't intersect with the polygon.""" + # Create test data + segments_data = { + TRACK_ID: ["track1", "track2"], + START_X: [20.0, 100.0], + START_Y: [20.0, 150.0], + END_X: [30.0, 110.0], + END_Y: [30.0, 160.0], + START_OCCURRENCE: [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + ], + END_OCCURRENCE: [ + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + ], + } + segments_df = DataFrame(segments_data) + + # Create a polygon that doesn't intersect with any segment + polygon = Polygon( + [ + Coordinate(0.0, 0.0), + Coordinate(10.0, 0.0), + Coordinate(10.0, 10.0), + Coordinate(0.0, 10.0), + Coordinate(0.0, 0.0), + ] + ) + + # Check intersections + result = check_polygon_intersections(segments_df, polygon) + + # Check that no segments intersect with the polygon + assert not result["intersects-polygon"].any() + + +def test_check_polygon_intersections_with_intersections() -> None: + """Test with segments that intersect with the polygon.""" + # Create test data + segments_data = { + TRACK_ID: ["track1", "track2", "track3"], + START_X: [5.0, 100.0, 0.0], + START_Y: [5.0, 150.0, 5.0], + END_X: [15.0, 110.0, 25.0], + END_Y: [15.0, 160.0, 15.0], + START_OCCURRENCE: [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + ], + END_OCCURRENCE: [ + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + ], + } + segments_df = DataFrame(segments_data) + + # Create a polygon that intersects with the first and third segments + polygon = Polygon( + [ + Coordinate(0.0, 0.0), + Coordinate(10.0, 0.0), + Coordinate(10.0, 10.0), + Coordinate(0.0, 10.0), + Coordinate(0.0, 0.0), + ] + ) + + # Check intersections + result = check_polygon_intersections(segments_df, polygon) + + # Check that the first and third segments intersect with the polygon + assert result.iloc[0]["intersects-polygon"] + assert not result.iloc[1]["intersects-polygon"] + assert result.iloc[2]["intersects-polygon"] + + # Check the count of intersecting segments + assert result["intersects-polygon"].sum() == 2 From aaf8f3ec210e16150128ad63e1314b22faedc656 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 16 May 2025 08:36:33 +0200 Subject: [PATCH 007/128] Use constant --- .../track_geometry_store/pandas_geometry_store.py | 7 ++++--- .../test_pandas_geometry_store.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 6f3742957..f659aed8d 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -21,6 +21,7 @@ INTERSECTION_X = "intersection_x" INTERSECTION_Y = "intersection_y" INTERSECTION_LINE_ID = "intersection_line_id" +INTERSECTS_POLYGON = "intersects-polygon" # Column names for intersection parameters DENOMINATOR = "denominator" @@ -340,7 +341,7 @@ def check_polygon_intersections( polygon (Polygon): The polygon to check intersections with Returns: - DataFrame: The input DataFrame with an additional column "intersects-polygon" + DataFrame: The input DataFrame with an additional column INTERSECTS_POLYGON indicating if the segment intersects with the polygon """ if segments_df.empty: @@ -350,7 +351,7 @@ def check_polygon_intersections( result_df = segments_df.copy() # Initialize the intersects-polygon column to False - result_df["intersects-polygon"] = False + result_df[INTERSECTS_POLYGON] = False # Get the coordinates of the polygon polygon_coordinates = polygon.coordinates @@ -370,6 +371,6 @@ def check_polygon_intersections( # Update the result DataFrame if intersects_df[INTERSECTS].any(): - result_df.loc[intersects_df[INTERSECTS], "intersects-polygon"] = True + result_df.loc[intersects_df[INTERSECTS], INTERSECTS_POLYGON] = True return result_df diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py index fc8270217..dc2cb64b8 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py @@ -13,6 +13,7 @@ INTERSECTION_X, INTERSECTION_Y, INTERSECTS, + INTERSECTS_POLYGON, OCCURRENCE, START_OCCURRENCE, START_X, @@ -307,7 +308,7 @@ def test_check_polygon_intersections_no_intersections() -> None: result = check_polygon_intersections(segments_df, polygon) # Check that no segments intersect with the polygon - assert not result["intersects-polygon"].any() + assert not result[INTERSECTS_POLYGON].any() def test_check_polygon_intersections_with_intersections() -> None: @@ -347,9 +348,9 @@ def test_check_polygon_intersections_with_intersections() -> None: result = check_polygon_intersections(segments_df, polygon) # Check that the first and third segments intersect with the polygon - assert result.iloc[0]["intersects-polygon"] - assert not result.iloc[1]["intersects-polygon"] - assert result.iloc[2]["intersects-polygon"] + assert result.iloc[0][INTERSECTS_POLYGON] + assert not result.iloc[1][INTERSECTS_POLYGON] + assert result.iloc[2][INTERSECTS_POLYGON] # Check the count of intersecting segments - assert result["intersects-polygon"].sum() == 2 + assert result[INTERSECTS_POLYGON].sum() == 2 From 4c5e17da184cf8b75a05d384e45d3f75c1867629 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 16 May 2025 08:47:21 +0200 Subject: [PATCH 008/128] Move polygon --- .../pandas_geometry_store.py | 93 ++++++++++++++++++- .../test_pandas_geometry_store.py | 3 +- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index f659aed8d..4dae800e7 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -1,6 +1,17 @@ +from dataclasses import dataclass +from typing import Any, Iterable, Sequence + from pandas import DataFrame -from OTAnalytics.domain.geometry import Polygon +from OTAnalytics.domain.common import DataclassValidation +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.section import Section, SectionId +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset.track_dataset import ( + IntersectionPoint, + TrackDataset, + TrackGeometryDataset, +) # Column names for track points TRACK_ID = "track-id" @@ -30,6 +41,30 @@ UB = "ub" +@dataclass(frozen=True) +class Polygon(DataclassValidation): + """A polygon is made up of line segments which form a closed polygonal chain. + + Args: + coordinates (list[Coordinate]): the coordinates defining the polygon + + Raises: + ValueError: if coordinates defining the polygon is less than 4 + ValueError: if coordinates do not define a closed polygonal chain + """ + + coordinates: list[Coordinate] + + def _validate(self) -> None: + if len(self.coordinates) < 2: + raise ValueError( + ( + "Number of coordinates to define a valid polygon must be " + f"greater equal two, but is {len(self.coordinates)}" + ) + ) + + def create_track_segments(df: DataFrame) -> DataFrame: """ Create track segments from a DataFrame containing track points. @@ -374,3 +409,59 @@ def check_polygon_intersections( result_df.loc[intersects_df[INTERSECTS], INTERSECTS_POLYGON] = True return result_df + + +class PandasTrackGeometryDataset(TrackGeometryDataset): + @property + def track_ids(self) -> set[str]: + # TODO Junie implement + raise NotImplementedError + + @property + def offset(self) -> RelativeOffsetCoordinate: + # TODO Do not implement + raise NotImplementedError + + @property + def empty(self) -> bool: + # TODO Junie implement + raise NotImplementedError + + @staticmethod + def from_track_dataset( + dataset: TrackDataset, offset: RelativeOffsetCoordinate + ) -> "TrackGeometryDataset": + # TODO Do not implement + raise NotImplementedError + + def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": + # TODO Do not implement + raise NotImplementedError + + def remove(self, ids: Sequence[str]) -> "TrackGeometryDataset": + # TODO Do not implement + raise NotImplementedError + + def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": + # TODO Do not implement + raise NotImplementedError + + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + # TODO Junie implement + raise NotImplementedError + + def intersection_points( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + # TODO Junie implement + raise NotImplementedError + + def contained_by_sections( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: + # TODO Do not implement + raise NotImplementedError + + def __eq__(self, other: Any) -> bool: + # TODO Junie implement + raise NotImplementedError diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py index dc2cb64b8..25592ce6f 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py @@ -4,7 +4,7 @@ from pandas import DataFrame from pytest import approx -from OTAnalytics.domain.geometry import Coordinate, Polygon +from OTAnalytics.domain.geometry import Coordinate from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( END_OCCURRENCE, END_X, @@ -19,6 +19,7 @@ START_X, START_Y, TRACK_ID, + Polygon, X, Y, check_polygon_intersections, From 8ff8b271c1a02ab4d4029ae037944f52e8f949fc Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 26 Jun 2025 11:03:20 +0200 Subject: [PATCH 009/128] Implement intersection with first line segment of section --- .../pandas_geometry_store.py | 226 +++++++++++++++++- 1 file changed, 214 insertions(+), 12 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 4dae800e7..e81887975 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -1,11 +1,11 @@ from dataclasses import dataclass -from typing import Any, Iterable, Sequence +from typing import Any, Iterable, Optional, Sequence from pandas import DataFrame from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.section import Section, SectionId +from OTAnalytics.domain.section import Section, SectionId, SectionType from OTAnalytics.domain.track import Track, TrackId from OTAnalytics.domain.track_dataset.track_dataset import ( IntersectionPoint, @@ -412,10 +412,45 @@ def check_polygon_intersections( class PandasTrackGeometryDataset(TrackGeometryDataset): + _segments_df: DataFrame + + def __init__(self, segments_df: Optional[DataFrame] = None): + """Initialize a PandasTrackGeometryDataset. + + Args: + segments_df (Optional[DataFrame], optional): DataFrame with track segments. + If None, an empty DataFrame will be created. Defaults to None. + """ + if segments_df is None: + self._segments_df = DataFrame() + else: + self._segments_df = segments_df.copy() + + # Ensure the DataFrame has the required columns + if not self._segments_df.empty and not all( + col in self._segments_df.columns + for col in [ + TRACK_ID, + START_X, + START_Y, + END_X, + END_Y, + START_OCCURRENCE, + END_OCCURRENCE, + ] + ): + raise ValueError("Segments DataFrame must have the required columns") + @property def track_ids(self) -> set[str]: - # TODO Junie implement - raise NotImplementedError + """Get track ids of tracks stored in dataset. + + Returns: + set[str]: the track ids stored. + """ + if self._segments_df.empty: + return set() + return set(self._segments_df[TRACK_ID].unique()) @property def offset(self) -> RelativeOffsetCoordinate: @@ -424,8 +459,12 @@ def offset(self) -> RelativeOffsetCoordinate: @property def empty(self) -> bool: - # TODO Junie implement - raise NotImplementedError + """Check if the dataset is empty. + + Returns: + bool: True if the dataset is empty, False otherwise. + """ + return self._segments_df.empty @staticmethod def from_track_dataset( @@ -447,14 +486,145 @@ def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": raise NotImplementedError def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: - # TODO Junie implement - raise NotImplementedError + """Return a set of tracks intersecting a set of sections. + + Args: + sections (list[Section]): the list of sections to intersect. + + Returns: + set[TrackId]: the track ids intersecting the given sections. + """ + if self.empty or not sections: + return set() + + # Create a set to store the track IDs that intersect with any section + intersecting_track_ids: set[TrackId] = set() + + # Check intersections for each section + for section in sections: + # Get the coordinates of the section + coordinates = section.get_coordinates() + + if ( + section.get_type() == SectionType.LINE + or section.get_type() == SectionType.CUTTING + ): + # For line sections, check if any track segment intersects with the line + start_x, start_y = coordinates[0].x, coordinates[0].y + end_x, end_y = coordinates[1].x, coordinates[1].y + + # Find intersections with the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, + ) + + # Add track IDs that intersect with the line to the result set + intersecting_segments = intersections[intersections[INTERSECTS]] + intersecting_track_ids.update(intersecting_segments[TRACK_ID].unique()) + elif section.get_type() == SectionType.AREA: + # For area sections, check if any track segment intersects with the + # polygon + polygon = Polygon(coordinates) + + # Check polygon intersections + intersections = check_polygon_intersections(self._segments_df, polygon) + + # Add track IDs that intersect with the polygon to the result set + intersecting_segments = intersections[intersections[INTERSECTS_POLYGON]] + intersecting_track_ids.update(intersecting_segments[TRACK_ID].unique()) + + return intersecting_track_ids def intersection_points( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - # TODO Junie implement - raise NotImplementedError + """Return the intersection points from tracks and the given sections. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + the intersection points. + """ + if self.empty or not sections: + return {} + + # Create a dictionary to store the intersection points for each track + result: dict[TrackId, list[tuple[SectionId, IntersectionPoint]]] = {} + + # Process only line sections + # (area sections don't have specific intersection points) + line_sections = [ + section + for section in sections + if section.get_type() == SectionType.LINE + or section.get_type() == SectionType.CUTTING + ] + + if not line_sections: + return {} + + # For each line section, find intersections with track segments + for section in line_sections: + # Get the coordinates of the section + coordinates = section.get_coordinates() + start_x, start_y = coordinates[0].x, coordinates[0].y + end_x, end_y = coordinates[1].x, coordinates[1].y + + # Find intersections with the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, + ) + + # Filter to only include segments that intersect with the line + intersecting_segments = intersections[intersections[INTERSECTS]] + + # Process each intersecting segment + for _, segment in intersecting_segments.iterrows(): + track_id = segment[TRACK_ID] + + # Calculate the relative position of the intersection point + # along the segment + segment_length_x = segment[END_X] - segment[START_X] + segment_length_y = segment[END_Y] - segment[START_Y] + + # Avoid division by zero + if segment_length_x == 0 and segment_length_y == 0: + continue + + # Calculate the relative position (0 to 1) along the segment + if segment_length_x != 0: + relative_position = ( + segment[INTERSECTION_X] - segment[START_X] + ) / segment_length_x + else: + relative_position = ( + segment[INTERSECTION_Y] - segment[START_Y] + ) / segment_length_y + + # Create an IntersectionPoint + # The upper_index is 1 because we're dealing with segments (2 points) + intersection_point = IntersectionPoint( + upper_index=1, relative_position=relative_position + ) + + # Add to the result dictionary + if track_id not in result: + result[track_id] = [] + result[track_id].append((section.id, intersection_point)) + + return result def contained_by_sections( self, sections: list[Section] @@ -463,5 +633,37 @@ def contained_by_sections( raise NotImplementedError def __eq__(self, other: Any) -> bool: - # TODO Junie implement - raise NotImplementedError + """Check if this dataset is equal to another dataset. + + Args: + other (Any): the other dataset to compare with. + + Returns: + bool: True if the datasets are equal, False otherwise. + """ + if not isinstance(other, PandasTrackGeometryDataset): + return False + + # If both are empty, they are equal + if self.empty and other.empty: + return True + + # If only one is empty, they are not equal + if self.empty != other.empty: + return False + + # Compare the DataFrames + # First, ensure both DataFrames have the same columns + if set(self._segments_df.columns) != set(other._segments_df.columns): + return False + + # Sort both DataFrames to ensure consistent comparison + self_df = self._segments_df.sort_values( + by=[TRACK_ID, START_OCCURRENCE] + ).reset_index(drop=True) + other_df = other._segments_df.sort_values( + by=[TRACK_ID, START_OCCURRENCE] + ).reset_index(drop=True) + + # Compare the DataFrames + return self_df.equals(other_df) From d19b5e1cf21b948d39100b691f03edb8ef03f72e Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 26 Jun 2025 11:14:20 +0200 Subject: [PATCH 010/128] Process all legs of a segment --- .../pandas_geometry_store.py | 133 +++++++++--------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index e81887975..3b0703dde 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -509,23 +509,27 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: section.get_type() == SectionType.LINE or section.get_type() == SectionType.CUTTING ): - # For line sections, check if any track segment intersects with the line - start_x, start_y = coordinates[0].x, coordinates[0].y - end_x, end_y = coordinates[1].x, coordinates[1].y - - # Find intersections with the line - intersections = find_line_intersections( - self._segments_df, - section.id.serialize(), - start_x, - start_y, - end_x, - end_y, - ) - - # Add track IDs that intersect with the line to the result set - intersecting_segments = intersections[intersections[INTERSECTS]] - intersecting_track_ids.update(intersecting_segments[TRACK_ID].unique()) + # For line sections, check if any track segment intersects with any leg + # of the line. A leg is formed by each consecutive pair of coordinates. + for i in range(len(coordinates) - 1): + start_x, start_y = coordinates[i].x, coordinates[i].y + end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y + + # Find intersections with this leg of the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, + ) + + # Add track IDs that intersect with the line to the result set + intersecting_segments = intersections[intersections[INTERSECTS]] + intersecting_track_ids.update( + intersecting_segments[TRACK_ID].unique() + ) elif section.get_type() == SectionType.AREA: # For area sections, check if any track segment intersects with the # polygon @@ -574,55 +578,58 @@ def intersection_points( for section in line_sections: # Get the coordinates of the section coordinates = section.get_coordinates() - start_x, start_y = coordinates[0].x, coordinates[0].y - end_x, end_y = coordinates[1].x, coordinates[1].y - - # Find intersections with the line - intersections = find_line_intersections( - self._segments_df, - section.id.serialize(), - start_x, - start_y, - end_x, - end_y, - ) - # Filter to only include segments that intersect with the line - intersecting_segments = intersections[intersections[INTERSECTS]] - - # Process each intersecting segment - for _, segment in intersecting_segments.iterrows(): - track_id = segment[TRACK_ID] - - # Calculate the relative position of the intersection point - # along the segment - segment_length_x = segment[END_X] - segment[START_X] - segment_length_y = segment[END_Y] - segment[START_Y] - - # Avoid division by zero - if segment_length_x == 0 and segment_length_y == 0: - continue - - # Calculate the relative position (0 to 1) along the segment - if segment_length_x != 0: - relative_position = ( - segment[INTERSECTION_X] - segment[START_X] - ) / segment_length_x - else: - relative_position = ( - segment[INTERSECTION_Y] - segment[START_Y] - ) / segment_length_y - - # Create an IntersectionPoint - # The upper_index is 1 because we're dealing with segments (2 points) - intersection_point = IntersectionPoint( - upper_index=1, relative_position=relative_position + # Process each leg of the section (consecutive pair of coordinates) + for i in range(len(coordinates) - 1): + start_x, start_y = coordinates[i].x, coordinates[i].y + end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y + + # Find intersections with this leg of the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, ) - # Add to the result dictionary - if track_id not in result: - result[track_id] = [] - result[track_id].append((section.id, intersection_point)) + # Filter to only include segments that intersect with the line + intersecting_segments = intersections[intersections[INTERSECTS]] + + # Process each intersecting segment + for _, segment in intersecting_segments.iterrows(): + track_id = segment[TRACK_ID] + + # Calculate the relative position of the intersection point + # along the segment + segment_length_x = segment[END_X] - segment[START_X] + segment_length_y = segment[END_Y] - segment[START_Y] + + # Avoid division by zero + if segment_length_x == 0 and segment_length_y == 0: + continue + + # Calculate the relative position (0 to 1) along the segment + if segment_length_x != 0: + relative_position = ( + segment[INTERSECTION_X] - segment[START_X] + ) / segment_length_x + else: + relative_position = ( + segment[INTERSECTION_Y] - segment[START_Y] + ) / segment_length_y + + # Create an IntersectionPoint + # The upper_index is 1 because we're dealing with segments + intersection_point = IntersectionPoint( + upper_index=1, relative_position=relative_position + ) + + # Add to the result dictionary + if track_id not in result: + result[track_id] = [] + result[track_id].append((section.id, intersection_point)) return result From 86e9cc7af9c037ef9295817f40077c03497a38df Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 13 Aug 2025 16:43:35 +0200 Subject: [PATCH 011/128] Fix type cast --- .../plugin_datastore/track_geometry_store/shapely_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py index 8aca02913..72140781d 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py @@ -95,7 +95,7 @@ class InvalidTrackGeometryDataset(Exception): def distance_on_track(point: Point, track_geom: GeometryCollection) -> float: distance = line_locate_point(track_geom, point) - return round(distance, NDIGITS_DISTANCE) + return float(round(distance, NDIGITS_DISTANCE)) class ShapelyTrackGeometryDataset(TrackGeometryDataset): From e7a2b29ddf5877edfed56b5c1e84a5a3e884e3d2 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 13 Aug 2025 16:43:46 +0200 Subject: [PATCH 012/128] Ensure offset is used --- .../pandas_geometry_store.py | 141 ++++++++++++++---- .../test_pandas_geometry_store.py | 114 ++++++++++++-- 2 files changed, 210 insertions(+), 45 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 3b0703dde..6b448007e 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -6,12 +6,14 @@ from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId, SectionType -from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track import H, Track, TrackId, W from OTAnalytics.domain.track_dataset.track_dataset import ( IntersectionPoint, TrackDataset, TrackGeometryDataset, ) +from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset # Column names for track points TRACK_ID = "track-id" @@ -24,8 +26,12 @@ END_OCCURRENCE = "end-occurrence" START_X = "start-x" START_Y = "start-y" +START_W = "start-w" +START_H = "start-h" END_X = "end-x" END_Y = "end-y" +END_W = "end-w" +END_H = "end-h" # Column names for intersections INTERSECTS = "intersects" @@ -43,14 +49,17 @@ @dataclass(frozen=True) class Polygon(DataclassValidation): - """A polygon is made up of line segments which form a closed polygonal chain. + """A polygon consists of an ordered sequence of coordinates. + + Note: + The coordinates are not required to form a closed chain in this representation. Args: - coordinates (list[Coordinate]): the coordinates defining the polygon + coordinates (list[Coordinate]): The coordinates defining the polygon. Must + contain at least two coordinates. Raises: - ValueError: if coordinates defining the polygon is less than 4 - ValueError: if coordinates do not define a closed polygonal chain + ValueError: If fewer than two coordinates are provided. """ coordinates: list[Coordinate] @@ -67,16 +76,16 @@ def _validate(self) -> None: def create_track_segments(df: DataFrame) -> DataFrame: """ - Create track segments from a DataFrame containing track points. + Create track segments from a DataFrame of track points. A track segment connects two consecutive points of a track. Args: - df (DataFrame): DataFrame with columns TRACK_ID, OCCURRENCE, X, Y + df (DataFrame): DataFrame with columns TRACK_ID, OCCURRENCE, X, Y, W, H. Returns: DataFrame: DataFrame with columns TRACK_ID, START_OCCURRENCE, END_OCCURRENCE, - START_X, START_Y, END_X, END_Y + START_X, START_Y, START_W, START_H, END_X, END_Y, END_W, END_H. """ if df.empty: return DataFrame() @@ -96,9 +105,13 @@ def create_track_segments(df: DataFrame) -> DataFrame: segments[END_OCCURRENCE] = data[OCCURRENCE] segments[END_X] = data[X] segments[END_Y] = data[Y] + segments[END_W] = data[W] + segments[END_H] = data[H] segments[START_OCCURRENCE] = grouped[OCCURRENCE].shift(1) segments[START_X] = grouped[X].shift(1) segments[START_Y] = grouped[Y].shift(1) + segments[START_W] = grouped[W].shift(1) + segments[START_H] = grouped[H].shift(1) # Remove rows where start values are NaN (first point of each track) segments = segments.dropna() @@ -109,6 +122,19 @@ def create_track_segments(df: DataFrame) -> DataFrame: return segments +def apply_offset(segments_df: DataFrame, offset: RelativeOffsetCoordinate) -> DataFrame: + """Apply an offset to the coordinates of a DataFrame containing track segments.""" + if segments_df.empty: + return segments_df + + segments_df = segments_df.copy() + segments_df[START_X] += segments_df[START_W] * offset.x + segments_df[START_Y] += segments_df[START_H] * offset.y + segments_df[END_X] += segments_df[END_W] * offset.x + segments_df[END_Y] += segments_df[END_H] * offset.y + return segments_df + + def calculate_intersection_parameters( segments_df: DataFrame, line_x1: float, @@ -175,23 +201,29 @@ def check_line_intersections( line_y1: float, line_x2: float, line_y2: float, + offset: RelativeOffsetCoordinate, ) -> DataFrame: """ Check if track segments intersect with a line. Args: - segments_df (DataFrame): DataFrame with track segments - line_x1 (float): X-coordinate of the line's start point - line_y1 (float): Y-coordinate of the line's start point - line_x2 (float): X-coordinate of the line's end point - line_y2 (float): Y-coordinate of the line's end point + segments_df (DataFrame): DataFrame with track segments. + line_x1 (float): X-coordinate of the line's start point. + line_y1 (float): Y-coordinate of the line's start point. + line_x2 (float): X-coordinate of the line's end point. + line_y2 (float): Y-coordinate of the line's end point. + offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints + before intersection checks. Returns: - DataFrame: Boolean mask indicating which segments intersect with the line + DataFrame: DataFrame with a single column INTERSECTS indicating whether each + segment intersects the line. """ if segments_df.empty: return segments_df + segments_df = apply_offset(segments_df, offset) + # Calculate intersection parameters params_df = calculate_intersection_parameters( segments_df, line_x1, line_y1, line_x2, line_y2 @@ -228,26 +260,31 @@ def calculate_intersection_points( line_y1: float, line_x2: float, line_y2: float, + offset: RelativeOffsetCoordinate, ) -> DataFrame: """ Calculate intersection points between track segments and a line. Args: - segments_df (DataFrame): DataFrame with track segments - line_x1 (float): X-coordinate of the line's start point - line_y1 (float): Y-coordinate of the line's start point - line_x2 (float): X-coordinate of the line's end point - line_y2 (float): Y-coordinate of the line's end point + segments_df (DataFrame): DataFrame with track segments. + line_x1 (float): X-coordinate of the line's start point. + line_y1 (float): Y-coordinate of the line's start point. + line_x2 (float): X-coordinate of the line's end point. + line_y2 (float): Y-coordinate of the line's end point. + offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints + before intersection calculation. Returns: DataFrame: DataFrame with columns INTERSECTION_X and INTERSECTION_Y containing - the coordinates of intersection points + the coordinates of intersection points. """ if segments_df.empty: return DataFrame( index=segments_df.index, columns=[INTERSECTION_X, INTERSECTION_Y] ) + segments_df = apply_offset(segments_df, offset) + # Calculate intersection parameters params_df = calculate_intersection_parameters( segments_df, line_x1, line_y1, line_x2, line_y2 @@ -302,6 +339,7 @@ def find_line_intersections( start_y: float, end_x: float, end_y: float, + offset: RelativeOffsetCoordinate, ) -> DataFrame: """ Find intersection points between track segments and a line. @@ -314,6 +352,8 @@ def find_line_intersections( start_y (float): Y-coordinate of the line's start point end_x (float): X-coordinate of the line's end point end_y (float): Y-coordinate of the line's end point + offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints + before intersection checks. Returns: DataFrame: The input DataFrame with additional columns for intersection @@ -337,13 +377,15 @@ def find_line_intersections( result_df[INTERSECTION_LINE_ID] = None # Check which segments intersect with the line - intersects_df = check_line_intersections(result_df, start_x, start_y, end_x, end_y) + intersects_df = check_line_intersections( + result_df, start_x, start_y, end_x, end_y, offset + ) # If there are any intersections if intersects_df[INTERSECTS].any(): # Calculate intersection points intersection_df = calculate_intersection_points( - result_df, start_x, start_y, end_x, end_y + result_df, start_x, start_y, end_x, end_y, offset ) # Update the result DataFrame with intersection information @@ -363,6 +405,7 @@ def find_line_intersections( def check_polygon_intersections( segments_df: DataFrame, polygon: Polygon, + offset: RelativeOffsetCoordinate, ) -> DataFrame: """ Check if track segments intersect with a polygon. @@ -372,12 +415,14 @@ def check_polygon_intersections( Args: segments_df (DataFrame): DataFrame with track segments (output of - create_track_segments) - polygon (Polygon): The polygon to check intersections with + create_track_segments). + polygon (Polygon): The polygon to check intersections with. + offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints + before intersection checks. Returns: DataFrame: The input DataFrame with an additional column INTERSECTS_POLYGON - indicating if the segment intersects with the polygon + indicating if the segment intersects with the polygon. """ if segments_df.empty: return segments_df @@ -401,7 +446,12 @@ def check_polygon_intersections( # Check which segments intersect with the current line segment intersects_df = check_line_intersections( - result_df, start_x, start_y, end_x, end_y + result_df, + start_x, + start_y, + end_x, + end_y, + offset, ) # Update the result DataFrame @@ -411,6 +461,12 @@ def check_polygon_intersections( return result_df +def get_section_offset(section: Section) -> RelativeOffsetCoordinate: + if offset := section.relative_offset_coordinates.get(EventType.SECTION_ENTER): + return offset + raise ValueError(f"Section {section.id} has no offset") + + class PandasTrackGeometryDataset(TrackGeometryDataset): _segments_df: DataFrame @@ -470,8 +526,29 @@ def empty(self) -> bool: def from_track_dataset( dataset: TrackDataset, offset: RelativeOffsetCoordinate ) -> "TrackGeometryDataset": - # TODO Do not implement - raise NotImplementedError + """Create a PandasTrackGeometryDataset from a TrackDataset. + + Args: + dataset (TrackDataset): The source track dataset. Must be a + PandasTrackDataset. + offset (RelativeOffsetCoordinate): Relative offset to apply to track points + before segment creation. Currently not applied in this implementation. + + Returns: + TrackGeometryDataset: A dataset containing the generated track segments. + + Raises: + ValueError: If the dataset is not a PandasTrackDataset. + """ + if isinstance(dataset, PandasTrackDataset): + data = dataset.get_data() + segments = create_track_segments(data) + return PandasTrackGeometryDataset(segments) + else: + raise ValueError( + "PandasTrackGeometryDataset can only be created from a " + "PandasTrackDataset" + ) def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": # TODO Do not implement @@ -504,6 +581,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: for section in sections: # Get the coordinates of the section coordinates = section.get_coordinates() + offset = get_section_offset(section) if ( section.get_type() == SectionType.LINE @@ -523,6 +601,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: start_y, end_x, end_y, + offset, ) # Add track IDs that intersect with the line to the result set @@ -536,7 +615,9 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: polygon = Polygon(coordinates) # Check polygon intersections - intersections = check_polygon_intersections(self._segments_df, polygon) + intersections = check_polygon_intersections( + self._segments_df, polygon, offset + ) # Add track IDs that intersect with the polygon to the result set intersecting_segments = intersections[intersections[INTERSECTS_POLYGON]] @@ -578,6 +659,7 @@ def intersection_points( for section in line_sections: # Get the coordinates of the section coordinates = section.get_coordinates() + offset = get_section_offset(section) # Process each leg of the section (consecutive pair of coordinates) for i in range(len(coordinates) - 1): @@ -592,6 +674,7 @@ def intersection_points( start_y, end_x, end_y, + offset, ) # Filter to only include segments that intersect with the line diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py index 25592ce6f..15fb7132e 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py @@ -1,12 +1,16 @@ from datetime import datetime import pandas as pd +import pytest from pandas import DataFrame from pytest import approx -from OTAnalytics.domain.geometry import Coordinate +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.track import H, W from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + END_H, END_OCCURRENCE, + END_W, END_X, END_Y, INTERSECTION_LINE_ID, @@ -15,7 +19,9 @@ INTERSECTS, INTERSECTS_POLYGON, OCCURRENCE, + START_H, START_OCCURRENCE, + START_W, START_X, START_Y, TRACK_ID, @@ -31,7 +37,9 @@ def test_find_line_intersections_empty_df() -> None: """Test that an empty DataFrame returns an empty DataFrame.""" df = DataFrame() - result = find_line_intersections(df, "line1", 0.0, 0.0, 10.0, 10.0) + result = find_line_intersections( + df, "line1", 0.0, 0.0, 10.0, 10.0, RelativeOffsetCoordinate(0.0, 0.0) + ) assert result.empty @@ -44,6 +52,10 @@ def test_find_line_intersections_no_intersections() -> None: START_Y: [15.0, 150.0], END_X: [20.0, 110.0], END_Y: [25.0, 160.0], + START_W: [0.0, 0.0], + START_H: [0.0, 0.0], + END_W: [0.0, 0.0], + END_H: [0.0, 0.0], START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), @@ -64,7 +76,13 @@ def test_find_line_intersections_no_intersections() -> None: # Find intersections result = find_line_intersections( - segments_df, line_id, start_x, start_y, end_x, end_y + segments_df, + line_id, + start_x, + start_y, + end_x, + end_y, + RelativeOffsetCoordinate(0.0, 0.0), ) # Check that no segments intersect @@ -74,15 +92,35 @@ def test_find_line_intersections_no_intersections() -> None: assert (result[INTERSECTION_LINE_ID].isna()).all() -def test_find_line_intersections_with_intersections() -> None: +@pytest.mark.parametrize( + "offset, expected_intersection", + [ + (RelativeOffsetCoordinate(0.0, 0.0), 30.0), + (RelativeOffsetCoordinate(0.5, 0.0), 33.0), + (RelativeOffsetCoordinate(1.0, 0.0), 36.0), + (RelativeOffsetCoordinate(0.0, 0.5), 33.0), + (RelativeOffsetCoordinate(0.5, 0.5), 36.0), + (RelativeOffsetCoordinate(1.0, 0.5), 39.0), + (RelativeOffsetCoordinate(0.0, 1.0), 36.0), + (RelativeOffsetCoordinate(0.5, 1.0), 39.0), + (RelativeOffsetCoordinate(1.0, 1.0), 42.0), + ], +) +def test_find_line_intersections_with_intersections( + offset: RelativeOffsetCoordinate, expected_intersection: float +) -> None: """Test with segments that intersect with the line.""" # Create test data segments_data = { TRACK_ID: ["track1", "track2"], - START_X: [15.0, 100.0], + START_X: [60.0, 100.0], START_Y: [0.0, 150.0], - END_X: [15.0, 110.0], - END_Y: [30.0, 160.0], + END_X: [0.0, 110.0], + END_Y: [60.0, 160.0], + START_W: [12.0, 12.0], + START_H: [12.0, 12.0], + END_W: [12.0, 12.0], + END_H: [12.0, 12.0], START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), @@ -98,18 +136,18 @@ def test_find_line_intersections_with_intersections() -> None: line_id = "intersecting_line" start_x = 0.0 start_y = 0.0 - end_x = 30.0 - end_y = 30.0 + end_x = 120.0 + end_y = 120.0 # Find intersections result = find_line_intersections( - segments_df, line_id, start_x, start_y, end_x, end_y + segments_df, line_id, start_x, start_y, end_x, end_y, offset ) # Check that the first segment intersects assert result.iloc[0][INTERSECTS] - assert result.iloc[0][INTERSECTION_X] == approx(15.0) - assert result.iloc[0][INTERSECTION_Y] == approx(15.0) + assert result.iloc[0][INTERSECTION_X] == approx(expected_intersection) + assert result.iloc[0][INTERSECTION_Y] == approx(expected_intersection) assert result.iloc[0][INTERSECTION_LINE_ID] == line_id # Check that the second segment doesn't intersect @@ -128,6 +166,10 @@ def test_find_line_intersections_multiple_intersections() -> None: START_Y: [10.0, 10.0, 40.0], END_X: [10.0, 110.0, 40.0], END_Y: [20.0, 20.0, 30.0], + START_W: [0.0, 0.0, 0.0], + START_H: [0.0, 0.0, 0.0], + END_W: [0.0, 0.0, 0.0], + END_H: [0.0, 0.0, 0.0], START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), @@ -150,7 +192,13 @@ def test_find_line_intersections_multiple_intersections() -> None: # Find intersections result = find_line_intersections( - segments_df, line_id, start_x, start_y, end_x, end_y + segments_df, + line_id, + start_x, + start_y, + end_x, + end_y, + RelativeOffsetCoordinate(0.0, 0.0), ) # Check that the first and third segments intersect @@ -189,6 +237,8 @@ def test_create_track_segments_single_track() -> None: ], X: [10.0, 20.0, 30.0], Y: [15.0, 25.0, 35.0], + W: [10.0, 20.0, 30.0], + H: [8.0, 16.0, 24.0], } df = DataFrame(data) @@ -204,8 +254,12 @@ def test_create_track_segments_single_track() -> None: assert result.iloc[0][END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) assert result.iloc[0][START_X] == 10.0 assert result.iloc[0][START_Y] == 15.0 + assert result.iloc[0][START_W] == 10.0 + assert result.iloc[0][START_H] == 8.0 assert result.iloc[0][END_X] == 20.0 assert result.iloc[0][END_Y] == 25.0 + assert result.iloc[0][END_W] == 20.0 + assert result.iloc[0][END_H] == 16.0 # Check second segment assert result.iloc[1][TRACK_ID] == "track1" @@ -213,8 +267,12 @@ def test_create_track_segments_single_track() -> None: assert result.iloc[1][END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 2) assert result.iloc[1][START_X] == 20.0 assert result.iloc[1][START_Y] == 25.0 + assert result.iloc[1][START_W] == 20.0 + assert result.iloc[1][START_H] == 16.0 assert result.iloc[1][END_X] == 30.0 assert result.iloc[1][END_Y] == 35.0 + assert result.iloc[1][END_W] == 30.0 + assert result.iloc[1][END_H] == 24.0 def test_create_track_segments_multiple_tracks() -> None: @@ -230,6 +288,8 @@ def test_create_track_segments_multiple_tracks() -> None: ], X: [10.0, 20.0, 100.0, 110.0], Y: [15.0, 25.0, 150.0, 160.0], + W: [5.0, 10.0, 20.0, 40.0], + H: [8.0, 16.0, 30.0, 60.0], } df = DataFrame(data) @@ -245,8 +305,12 @@ def test_create_track_segments_multiple_tracks() -> None: assert track1_segment[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) assert track1_segment[START_X] == 10.0 assert track1_segment[START_Y] == 15.0 + assert track1_segment[START_W] == 5.0 + assert track1_segment[START_H] == 8.0 assert track1_segment[END_X] == 20.0 assert track1_segment[END_Y] == 25.0 + assert track1_segment[END_W] == 10.0 + assert track1_segment[END_H] == 16.0 # Check track2 segment track2_segment = result[result[TRACK_ID] == "track2"].iloc[0] @@ -254,8 +318,12 @@ def test_create_track_segments_multiple_tracks() -> None: assert track2_segment[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) assert track2_segment[START_X] == 100.0 assert track2_segment[START_Y] == 150.0 + assert track2_segment[START_W] == 20.0 + assert track2_segment[START_H] == 30.0 assert track2_segment[END_X] == 110.0 assert track2_segment[END_Y] == 160.0 + assert track2_segment[END_W] == 40.0 + assert track2_segment[END_H] == 60.0 def test_check_polygon_intersections_empty_df() -> None: @@ -270,7 +338,9 @@ def test_check_polygon_intersections_empty_df() -> None: Coordinate(0.0, 0.0), ] ) - result = check_polygon_intersections(df, polygon) + result = check_polygon_intersections( + df, polygon, RelativeOffsetCoordinate(0.0, 0.0) + ) assert result.empty @@ -283,6 +353,10 @@ def test_check_polygon_intersections_no_intersections() -> None: START_Y: [20.0, 150.0], END_X: [30.0, 110.0], END_Y: [30.0, 160.0], + START_W: [0.0, 0.0], + START_H: [0.0, 0.0], + END_W: [0.0, 0.0], + END_H: [0.0, 0.0], START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), @@ -306,7 +380,9 @@ def test_check_polygon_intersections_no_intersections() -> None: ) # Check intersections - result = check_polygon_intersections(segments_df, polygon) + result = check_polygon_intersections( + segments_df, polygon, RelativeOffsetCoordinate(0.0, 0.0) + ) # Check that no segments intersect with the polygon assert not result[INTERSECTS_POLYGON].any() @@ -321,6 +397,10 @@ def test_check_polygon_intersections_with_intersections() -> None: START_Y: [5.0, 150.0, 5.0], END_X: [15.0, 110.0, 25.0], END_Y: [15.0, 160.0, 15.0], + START_W: [0.0, 0.0, 0.0], + START_H: [0.0, 0.0, 0.0], + END_W: [0.0, 0.0, 0.0], + END_H: [0.0, 0.0, 0.0], START_OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 0), @@ -346,7 +426,9 @@ def test_check_polygon_intersections_with_intersections() -> None: ) # Check intersections - result = check_polygon_intersections(segments_df, polygon) + result = check_polygon_intersections( + segments_df, polygon, RelativeOffsetCoordinate(0.0, 0.0) + ) # Check that the first and third segments intersect with the polygon assert result.iloc[0][INTERSECTS_POLYGON] From 44e7e6c809388d5e7e0594d0800b97d72257d6ae Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 13 Aug 2025 20:08:25 +0200 Subject: [PATCH 013/128] Add pure pandas and 2h test data --- tests/benchmark_otanalytics.py | 39 ++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index d3074f81c..ad5324e25 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -55,6 +55,9 @@ FilteredPythonTrackDataset, PythonTrackDataset, ) +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + PandasTrackGeometryDataset, +) from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, ) @@ -76,6 +79,7 @@ PYTHON = "PYTHON" PANDAS = "PANDAS" +PURE_PANDAS = "PANDAS" CURRENT_DATASET_TYPE = PANDAS EXCLUDE_FILTER = [ @@ -208,6 +212,9 @@ def provide_track_repository( elif dataset_type == PANDAS: repository = TrackRepository(self.provide_pandas_track_dataset()) parser = OttrkParser(self.provide_pandas_detection_parser()) + elif dataset_type == PURE_PANDAS: + repository = TrackRepository(self.provide_pure_pandas_track_dataset()) + parser = OttrkParser(self.provide_pure_pandas_detection_parser()) else: raise ValueError(f"Unknown dataset type {dataset_type}") detection_metadata = _fill_track_repository(parser, repository, track_files) @@ -222,6 +229,11 @@ def provide_pandas_track_dataset(self) -> TrackDataset: self._exclude_classes, ) + def provide_pandas_detection_parser(self) -> PandasDetectionParser: + return PandasDetectionParser( + PandasByMaxConfidence(), ShapelyTrackGeometryDataset.from_track_dataset + ) + def provide_python_track_dataset(self) -> TrackDataset: return FilteredPythonTrackDataset( PythonTrackDataset(ShapelyTrackGeometryDataset.from_track_dataset), @@ -238,9 +250,18 @@ def provide_python_detection_parser( ShapelyTrackGeometryDataset.from_track_dataset, ) - def provide_pandas_detection_parser(self) -> PandasDetectionParser: + def provide_pure_pandas_track_dataset(self) -> TrackDataset: + return FilterByClassPandasTrackDataset( + PandasTrackDataset.from_list( + [], PandasTrackGeometryDataset.from_track_dataset + ), + self._include_classes, + self._exclude_classes, + ) + + def provide_pure_pandas_detection_parser(self) -> PandasDetectionParser: return PandasDetectionParser( - PandasByMaxConfidence(), ShapelyTrackGeometryDataset.from_track_dataset + PandasByMaxConfidence(), PandasTrackGeometryDataset.from_track_dataset ) def counting_specification(self, save_dir: Path) -> CountingSpecificationDto: @@ -280,6 +301,8 @@ def get_track_parser(self, dataset_type: str = CURRENT_DATASET_TYPE) -> TrackPar ) elif dataset_type == PANDAS: return OttrkParser(self.provide_pandas_detection_parser()) + elif dataset_type == PURE_PANDAS: + return OttrkParser(self.provide_pure_pandas_detection_parser()) else: raise ValueError(f"Unknown dataset type {dataset_type}") @@ -441,6 +464,18 @@ def test_15min( warmup_rounds=self.WARMUP_ROUNDS, ) + def test_2hours( + self, benchmark: BenchmarkFixture, use_case_provider_2hours: UseCaseProvider + ) -> None: + use_case = use_case_provider_2hours.get_tracks_intersecting_sections() + benchmark.pedantic( + use_case, + args=(use_case_provider_2hours.sections,), + rounds=self.ROUNDS, + iterations=self.ITERATIONS, + warmup_rounds=self.WARMUP_ROUNDS, + ) + def test_15min_filtered( self, benchmark: BenchmarkFixture, From 0b9b88d0d73f0ad2dbca801325eb510e05fbbbf0 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 13 Aug 2025 21:22:46 +0200 Subject: [PATCH 014/128] Add 4h benchmark --- tests/benchmark_otanalytics.py | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index ad5324e25..b7dfffd6f 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -336,6 +336,28 @@ def track_files_2hours(test_data_dir: Path) -> list[Path]: ] +@pytest.fixture(scope="module") +def track_files_4hours(test_data_dir: Path) -> list[Path]: + return [ + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-00-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-15-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-30-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-45-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-00-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-15-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-30-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-45-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-00-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-15-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-30-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-45-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-00-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-15-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-30-00.ottrk"), + Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-45-00.ottrk"), + ] + + @pytest.fixture(scope="module") def otflow_file(test_data_dir: Path) -> Path: return test_data_dir / Path("OTCamera19_FR20_2023-05-24.otflow") @@ -387,6 +409,13 @@ def use_case_provider_2hours( return UseCaseProvider(otflow_file, track_files_2hours, str(test_data_tmp_dir)) +@pytest.fixture +def use_case_provider_4hours( + otflow_file: Path, track_files_4hours: list[Path], test_data_tmp_dir: Path +) -> UseCaseProvider: + return UseCaseProvider(otflow_file, track_files_4hours, str(test_data_tmp_dir)) + + @pytest.fixture def use_case_provider_15min_filtered( otflow_file: Path, track_file_15min: Path, test_data_tmp_dir: Path @@ -476,6 +505,18 @@ def test_2hours( warmup_rounds=self.WARMUP_ROUNDS, ) + def test_4hours( + self, benchmark: BenchmarkFixture, use_case_provider_4hours: UseCaseProvider + ) -> None: + use_case = use_case_provider_4hours.get_tracks_intersecting_sections() + benchmark.pedantic( + use_case, + args=(use_case_provider_4hours.sections,), + rounds=self.ROUNDS, + iterations=self.ITERATIONS, + warmup_rounds=self.WARMUP_ROUNDS, + ) + def test_15min_filtered( self, benchmark: BenchmarkFixture, From cc530d8bb0c98392e3ab2eab990d57cd7083ea26 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 14:20:49 +0200 Subject: [PATCH 015/128] Extend benchmark to check pure pandas implementation --- tests/benchmark_otanalytics.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index b7dfffd6f..0ea7b949f 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -1,4 +1,5 @@ from datetime import datetime, timezone +from functools import cached_property from pathlib import Path from typing import Callable, Iterable @@ -79,8 +80,8 @@ PYTHON = "PYTHON" PANDAS = "PANDAS" -PURE_PANDAS = "PANDAS" -CURRENT_DATASET_TYPE = PANDAS +PURE_PANDAS = "PURE_PANDAS" +CURRENT_DATASET_TYPE = PURE_PANDAS EXCLUDE_FILTER = [ OtcClasses.PEDESTRIAN, @@ -107,6 +108,22 @@ def _fill_track_repository( return DetectionMetadata(frozenset(detection_classes)) +class BenchmarkOtAnalyticsStarter(OtAnalyticsCliApplicationStarter): + def __init__( + self, + run_config: RunConfiguration, + dataset_type: str, + ) -> None: + super().__init__(run_config) + self._dataset_type = dataset_type + + @cached_property + def track_geometry_factory(self) -> TRACK_GEOMETRY_FACTORY: + if self._dataset_type == PURE_PANDAS: + return PandasTrackGeometryDataset.from_track_dataset + return ShapelyTrackGeometryDataset.from_track_dataset + + class UseCaseProvider: GEOMETRY_FACTORY: TRACK_GEOMETRY_FACTORY = ( ShapelyTrackGeometryDataset.from_track_dataset @@ -146,7 +163,7 @@ def __init__( self._include_classes: frozenset[str] = frozenset() self._exclude_classes: frozenset[str] = frozenset() self._flow_parser = OtFlowParser() - self._starter = OtAnalyticsCliApplicationStarter(self.run_config) + self._starter = BenchmarkOtAnalyticsStarter(self.run_config, dataset_type) track_repository, detection_metadata = self.provide_track_repository( self._ottrk_files, dataset_type ) From 15ee577780899a74a3b4222071b1752326410606 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 14:24:50 +0200 Subject: [PATCH 016/128] Optimize calculation --- .../pandas_geometry_store.py | 222 ++++++++++++------ 1 file changed, 152 insertions(+), 70 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 6b448007e..661875817 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -90,34 +90,30 @@ def create_track_segments(df: DataFrame) -> DataFrame: if df.empty: return DataFrame() - # Make a copy to avoid modifying the original DataFrame - data = df.copy() + # Create MultiIndex and sort once + df_indexed = df.set_index([TRACK_ID, OCCURRENCE]).sort_index() - # Ensure the DataFrame is sorted by track-id and occurrence - data = data.sort_values(by=[TRACK_ID, OCCURRENCE]) + # Group by level 0 (TRACK_ID) - this is very fast on sorted MultiIndex + grouped = df_indexed.groupby(level=0) - # Group by track-id to create segments - grouped = data.groupby(TRACK_ID) - - # Create a new DataFrame with shifted values to get start and end points + # Create segments DataFrame segments = DataFrame() - segments[TRACK_ID] = data[TRACK_ID] - segments[END_OCCURRENCE] = data[OCCURRENCE] - segments[END_X] = data[X] - segments[END_Y] = data[Y] - segments[END_W] = data[W] - segments[END_H] = data[H] + segments[TRACK_ID] = df_indexed.index.get_level_values(0) + segments[END_OCCURRENCE] = df_indexed.index.get_level_values(1) + segments[END_X] = df_indexed[X] + segments[END_Y] = df_indexed[Y] + segments[END_W] = df_indexed[W] + segments[END_H] = df_indexed[H] + + # Shift operations are faster on sorted groups segments[START_OCCURRENCE] = grouped[OCCURRENCE].shift(1) segments[START_X] = grouped[X].shift(1) segments[START_Y] = grouped[Y].shift(1) segments[START_W] = grouped[W].shift(1) segments[START_H] = grouped[H].shift(1) - # Remove rows where start values are NaN (first point of each track) - segments = segments.dropna() - - # Reset index for clean output - segments = segments.reset_index(drop=True) + # Remove NaN rows and reset index + segments = segments.dropna().reset_index(drop=True) return segments @@ -141,6 +137,7 @@ def calculate_intersection_parameters( line_y1: float, line_x2: float, line_y2: float, + offset: RelativeOffsetCoordinate, ) -> DataFrame: """ Calculate parameters needed for line intersection calculations. @@ -162,10 +159,16 @@ def calculate_intersection_parameters( # Initialize result DataFrame result_df = DataFrame(index=segments_df.index) + # Apply offset on-the-fly (no mutation/copy) + sxa = segments_df[START_X] + segments_df[START_W] * offset.x + sya = segments_df[START_Y] + segments_df[START_H] * offset.y + exa = segments_df[END_X] + segments_df[END_W] * offset.x + eya = segments_df[END_Y] + segments_df[END_H] * offset.y + # Calculate denominator for all segments at once - result_df[DENOMINATOR] = (line_y2 - line_y1) * ( - segments_df[END_X] - segments_df[START_X] - ) - (line_x2 - line_x1) * (segments_df[END_Y] - segments_df[START_Y]) + result_df[DENOMINATOR] = (line_y2 - line_y1) * (exa - sxa) - (line_x2 - line_x1) * ( + eya - sya + ) # Filter out segments where lines are parallel (denominator == 0) result_df[NON_PARALLEL] = result_df[DENOMINATOR] != 0 @@ -177,15 +180,12 @@ def calculate_intersection_parameters( if result_df[NON_PARALLEL].any(): # Calculate the parameters for the intersection points ua_values = ( - (line_x2 - line_x1) * (segments_df[START_Y] - line_y1) - - (line_y2 - line_y1) * (segments_df[START_X] - line_x1) + (line_x2 - line_x1) * (sya - line_y1) + - (line_y2 - line_y1) * (sxa - line_x1) ) / result_df[DENOMINATOR] ub_values = ( - (segments_df[END_X] - segments_df[START_X]) - * (segments_df[START_Y] - line_y1) - - (segments_df[END_Y] - segments_df[START_Y]) - * (segments_df[START_X] - line_x1) + (exa - sxa) * (sya - line_y1) - (eya - sya) * (sxa - line_x1) ) / result_df[DENOMINATOR] # Update ua and ub columns for non-parallel segments @@ -220,25 +220,56 @@ def check_line_intersections( segment intersects the line. """ if segments_df.empty: - return segments_df + return DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) + + # Bounding box of the line + line_min_x = min(line_x1, line_x2) + line_max_x = max(line_x1, line_x2) + line_min_y = min(line_y1, line_y2) + line_max_y = max(line_y1, line_y2) + + # Segment endpoints adjusted by offset (no copy/mutation) + sxa = segments_df[START_X] + segments_df[START_W] * offset.x + sya = segments_df[START_Y] + segments_df[START_H] * offset.y + exa = segments_df[END_X] + segments_df[END_W] * offset.x + eya = segments_df[END_Y] + segments_df[END_H] * offset.y + + # Segment bounding boxes + seg_min_x = sxa.where(sxa <= exa, exa) + seg_max_x = sxa.where(sxa >= exa, exa) + seg_min_y = sya.where(sya <= eya, eya) + seg_max_y = sya.where(sya >= eya, eya) + + # Candidate segments whose bbox intersects the line bbox + candidate_mask = ( + (seg_max_x >= line_min_x) + & (seg_min_x <= line_max_x) + & (seg_max_y >= line_min_y) + & (seg_min_y <= line_max_y) + ) + + # Prepare result DataFrame (all False by default) + intersects_df = DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) - segments_df = apply_offset(segments_df, offset) + if not candidate_mask.any(): + return intersects_df - # Calculate intersection parameters + # Calculate intersection parameters for candidates only + candidate_df = segments_df.loc[candidate_mask] params_df = calculate_intersection_parameters( - segments_df, line_x1, line_y1, line_x2, line_y2 + candidate_df, line_x1, line_y1, line_x2, line_y2, offset ) - # If all segments are parallel or ua/ub couldn't be calculated + # If all candidate segments are parallel or ua/ub couldn't be calculated if ( not params_df[NON_PARALLEL].any() or params_df[UA].isna().all() or params_df[UB].isna().all() ): - return DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) + return intersects_df - # Create a mask for valid intersections (intersection point is on both line - # segments) + # Create a mask for valid intersections + # (intersection point is on both line segments) valid_intersection_mask = ( (0 <= params_df[UA]) & (params_df[UA] <= 1) @@ -247,9 +278,10 @@ def check_line_intersections( & params_df[NON_PARALLEL] ) - # Create a DataFrame with the intersection results - intersects_df = DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) - intersects_df.loc[valid_intersection_mask, INTERSECTS] = True + # Update the result DataFrame with True where intersections occur + if valid_intersection_mask.any(): + valid_idx = params_df.index[valid_intersection_mask] + intersects_df.loc[valid_idx, INTERSECTS] = True return intersects_df @@ -283,11 +315,9 @@ def calculate_intersection_points( index=segments_df.index, columns=[INTERSECTION_X, INTERSECTION_Y] ) - segments_df = apply_offset(segments_df, offset) - - # Calculate intersection parameters + # Calculate intersection parameters (offset applied on-the-fly) params_df = calculate_intersection_parameters( - segments_df, line_x1, line_y1, line_x2, line_y2 + segments_df, line_x1, line_y1, line_x2, line_y2, offset ) # Initialize DataFrame for intersection coordinates @@ -313,13 +343,15 @@ def calculate_intersection_points( ) if valid_intersection_mask.any(): + # Adjusted segment endpoints with offset + sxa = segments_df[START_X] + segments_df[START_W] * offset.x + sya = segments_df[START_Y] + segments_df[START_H] * offset.y + exa = segments_df[END_X] + segments_df[END_W] * offset.x + eya = segments_df[END_Y] + segments_df[END_H] * offset.y + # Calculate the intersection points for valid intersections - intersection_x = segments_df[START_X] + params_df[UA] * ( - segments_df[END_X] - segments_df[START_X] - ) - intersection_y = segments_df[START_Y] + params_df[UA] * ( - segments_df[END_Y] - segments_df[START_Y] - ) + intersection_x = sxa + params_df[UA] * (exa - sxa) + intersection_y = sya + params_df[UA] * (eya - sya) # Update the result DataFrame with intersection coordinates intersection_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_x[ @@ -339,7 +371,7 @@ def find_line_intersections( start_y: float, end_x: float, end_y: float, - offset: RelativeOffsetCoordinate, + offset: RelativeOffsetCoordinate | None = None, ) -> DataFrame: """ Find intersection points between track segments and a line. @@ -367,8 +399,12 @@ def find_line_intersections( if segments_df.empty: return segments_df - # Create a copy to avoid modifying the original DataFrame - result_df = segments_df.copy() + # Default offset if not provided + if offset is None: + offset = RelativeOffsetCoordinate(0.0, 0.0) + + # Use a shallow copy to avoid duplicating data blocks + result_df = segments_df.copy(deep=False) # Initialize new columns result_df[INTERSECTS] = False @@ -378,24 +414,30 @@ def find_line_intersections( # Check which segments intersect with the line intersects_df = check_line_intersections( - result_df, start_x, start_y, end_x, end_y, offset + segments_df, start_x, start_y, end_x, end_y, offset ) # If there are any intersections if intersects_df[INTERSECTS].any(): - # Calculate intersection points + valid_intersection_mask = intersects_df[INTERSECTS] + + # Calculate intersection points only for intersecting segments intersection_df = calculate_intersection_points( - result_df, start_x, start_y, end_x, end_y, offset + segments_df.loc[valid_intersection_mask], + start_x, + start_y, + end_x, + end_y, + offset, ) # Update the result DataFrame with intersection information - valid_intersection_mask = intersects_df[INTERSECTS] result_df.loc[valid_intersection_mask, INTERSECTS] = True - result_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_df.loc[ - valid_intersection_mask, INTERSECTION_X + result_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_df[ + INTERSECTION_X ] - result_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_df.loc[ - valid_intersection_mask, INTERSECTION_Y + result_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_df[ + INTERSECTION_Y ] result_df.loc[valid_intersection_mask, INTERSECTION_LINE_ID] = line_id @@ -427,8 +469,8 @@ def check_polygon_intersections( if segments_df.empty: return segments_df - # Create a copy to avoid modifying the original DataFrame - result_df = segments_df.copy() + # Shallow copy to add result column without duplicating data + result_df = segments_df.copy(deep=False) # Initialize the intersects-polygon column to False result_df[INTERSECTS_POLYGON] = False @@ -436,6 +478,35 @@ def check_polygon_intersections( # Get the coordinates of the polygon polygon_coordinates = polygon.coordinates + # Polygon bounding box + poly_min_x = min(c.x for c in polygon_coordinates) + poly_max_x = max(c.x for c in polygon_coordinates) + poly_min_y = min(c.y for c in polygon_coordinates) + poly_max_y = max(c.y for c in polygon_coordinates) + + # Segment endpoints adjusted by offset (no copy/mutation) + sxa = segments_df[START_X] + segments_df[START_W] * offset.x + sya = segments_df[START_Y] + segments_df[START_H] * offset.y + exa = segments_df[END_X] + segments_df[END_W] * offset.x + eya = segments_df[END_Y] + segments_df[END_H] * offset.y + + # Segment bounding boxes + seg_min_x = sxa.where(sxa <= exa, exa) + seg_max_x = sxa.where(sxa >= exa, exa) + seg_min_y = sya.where(sya <= eya, eya) + seg_max_y = sya.where(sya >= eya, eya) + + # Pre-filter segments whose bbox intersects polygon bbox + candidate_mask = ( + (seg_max_x >= poly_min_x) + & (seg_min_x <= poly_max_x) + & (seg_max_y >= poly_min_y) + & (seg_min_y <= poly_max_y) + ) + + if not candidate_mask.any(): + return result_df + # Check intersections with each line segment of the polygon for i in range(len(polygon_coordinates) - 1): # Get the start and end points of the current line segment @@ -444,19 +515,30 @@ def check_polygon_intersections( end_x = polygon_coordinates[i + 1].x end_y = polygon_coordinates[i + 1].y + # Further prefilter by line bbox + line_min_x = min(start_x, end_x) + line_max_x = max(start_x, end_x) + line_min_y = min(start_y, end_y) + line_max_y = max(start_y, end_y) + cand2_mask = ( + (seg_max_x >= line_min_x) + & (seg_min_x <= line_max_x) + & (seg_max_y >= line_min_y) + & (seg_min_y <= line_max_y) + & candidate_mask + ) + if not cand2_mask.any(): + continue + # Check which segments intersect with the current line segment intersects_df = check_line_intersections( - result_df, - start_x, - start_y, - end_x, - end_y, - offset, + segments_df.loc[cand2_mask], start_x, start_y, end_x, end_y, offset ) - # Update the result DataFrame + # Update the result DataFrame using indices of True values if intersects_df[INTERSECTS].any(): - result_df.loc[intersects_df[INTERSECTS], INTERSECTS_POLYGON] = True + true_idx = intersects_df.index[intersects_df[INTERSECTS]] + result_df.loc[true_idx, INTERSECTS_POLYGON] = True return result_df From b5941819ce63c7ea266bc8035a9405b4201bb074 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 14:43:16 +0200 Subject: [PATCH 017/128] Add offset --- .../track_geometry_store/pandas_geometry_store.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 661875817..67cb37628 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -552,13 +552,16 @@ def get_section_offset(section: Section) -> RelativeOffsetCoordinate: class PandasTrackGeometryDataset(TrackGeometryDataset): _segments_df: DataFrame - def __init__(self, segments_df: Optional[DataFrame] = None): + def __init__( + self, offset: RelativeOffsetCoordinate, segments_df: Optional[DataFrame] = None + ): """Initialize a PandasTrackGeometryDataset. Args: segments_df (Optional[DataFrame], optional): DataFrame with track segments. If None, an empty DataFrame will be created. Defaults to None. """ + self._offset = offset if segments_df is None: self._segments_df = DataFrame() else: @@ -592,8 +595,7 @@ def track_ids(self) -> set[str]: @property def offset(self) -> RelativeOffsetCoordinate: - # TODO Do not implement - raise NotImplementedError + return self._offset @property def empty(self) -> bool: @@ -625,7 +627,7 @@ def from_track_dataset( if isinstance(dataset, PandasTrackDataset): data = dataset.get_data() segments = create_track_segments(data) - return PandasTrackGeometryDataset(segments) + return PandasTrackGeometryDataset(offset, segments) else: raise ValueError( "PandasTrackGeometryDataset can only be created from a " From 706357bcb5aa252edc134ac07edfce272af52357 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 20:52:42 +0200 Subject: [PATCH 018/128] Fix missing offset in intersection coordinate calculation --- .../pandas_geometry_store.py | 64 +++++++++++++------ 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 67cb37628..ba71151ab 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -1,3 +1,4 @@ +import math from dataclasses import dataclass from typing import Any, Iterable, Optional, Sequence @@ -91,29 +92,33 @@ def create_track_segments(df: DataFrame) -> DataFrame: return DataFrame() # Create MultiIndex and sort once - df_indexed = df.set_index([TRACK_ID, OCCURRENCE]).sort_index() + if TRACK_ID in df.columns or OCCURRENCE in df.columns: + df_indexed = df.reset_index().set_index([TRACK_ID, OCCURRENCE]).sort_index() + else: + df_indexed = df.sort_index() + df_indexed[END_OCCURRENCE] = df_indexed.index.get_level_values(1) # Group by level 0 (TRACK_ID) - this is very fast on sorted MultiIndex grouped = df_indexed.groupby(level=0) # Create segments DataFrame - segments = DataFrame() + segments = DataFrame(index=df_indexed.index) segments[TRACK_ID] = df_indexed.index.get_level_values(0) - segments[END_OCCURRENCE] = df_indexed.index.get_level_values(1) + segments[END_OCCURRENCE] = df_indexed[END_OCCURRENCE] segments[END_X] = df_indexed[X] segments[END_Y] = df_indexed[Y] segments[END_W] = df_indexed[W] segments[END_H] = df_indexed[H] # Shift operations are faster on sorted groups - segments[START_OCCURRENCE] = grouped[OCCURRENCE].shift(1) + segments[START_OCCURRENCE] = grouped[END_OCCURRENCE].shift(1) segments[START_X] = grouped[X].shift(1) segments[START_Y] = grouped[Y].shift(1) segments[START_W] = grouped[W].shift(1) segments[START_H] = grouped[H].shift(1) # Remove NaN rows and reset index - segments = segments.dropna().reset_index(drop=True) + segments = segments.dropna().reset_index() return segments @@ -565,22 +570,31 @@ def __init__( if segments_df is None: self._segments_df = DataFrame() else: - self._segments_df = segments_df.copy() + self._segments_df = segments_df # Ensure the DataFrame has the required columns - if not self._segments_df.empty and not all( - col in self._segments_df.columns + missing_columns = [ + col for col in [ TRACK_ID, START_X, START_Y, + START_W, + START_H, END_X, END_Y, + END_W, + END_H, START_OCCURRENCE, END_OCCURRENCE, ] - ): - raise ValueError("Segments DataFrame must have the required columns") + if col not in self._segments_df.columns + ] + if not self._segments_df.empty and len(missing_columns) > 0: + raise ValueError( + "Segments DataFrame must have the required columns. " + f"Following columns are missing: {missing_columns}" + ) @property def track_ids(self) -> set[str]: @@ -770,22 +784,30 @@ def intersection_points( # Calculate the relative position of the intersection point # along the segment - segment_length_x = segment[END_X] - segment[START_X] - segment_length_y = segment[END_Y] - segment[START_Y] - + segment_length_x = (segment[END_X] + segment[END_W] * offset.x) - ( + segment[START_X] + segment[START_W] * offset.x + ) + segment_length_y = (segment[END_Y] + segment[END_H] * offset.y) - ( + segment[START_Y] + segment[START_H] * offset.y + ) + segment_length = math.sqrt( + segment_length_x**2 + segment_length_y**2 + ) # Avoid division by zero if segment_length_x == 0 and segment_length_y == 0: continue # Calculate the relative position (0 to 1) along the segment - if segment_length_x != 0: - relative_position = ( - segment[INTERSECTION_X] - segment[START_X] - ) / segment_length_x - else: - relative_position = ( - segment[INTERSECTION_Y] - segment[START_Y] - ) / segment_length_y + intersection_length_x = segment[INTERSECTION_X] - ( + segment[START_X] + segment[START_W] * offset.x + ) + intersection_length_y = segment[INTERSECTION_Y] - ( + segment[START_Y] + segment[START_H] * offset.y + ) + intersection_length = math.sqrt( + intersection_length_x**2 + intersection_length_y**2 + ) + relative_position = intersection_length / segment_length # Create an IntersectionPoint # The upper_index is 1 because we're dealing with segments From aa3bc49081d2c4de2c5ba2d278fa6a417a18c7c3 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 20:53:49 +0200 Subject: [PATCH 019/128] Update todo comments --- .../track_geometry_store/pandas_geometry_store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index ba71151ab..6d34e85f9 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -649,15 +649,15 @@ def from_track_dataset( ) def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": - # TODO Do not implement + # TODO convert to segments and append segments to dataframe raise NotImplementedError def remove(self, ids: Sequence[str]) -> "TrackGeometryDataset": - # TODO Do not implement + # TODO remove all segments with given ids raise NotImplementedError def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": - # TODO Do not implement + # TODO filter segments by track ids raise NotImplementedError def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: From 16f1255c07a5c7d44514261a2e3eb8e9fba28002 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 21:02:36 +0200 Subject: [PATCH 020/128] Fix different track id classes and add further todos --- OTAnalytics/plugin_datastore/track_store.py | 47 ++++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 40ba4a46c..765fad863 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -248,6 +248,14 @@ def get_data(self) -> DataFrame: pass +def unpack(track_id: TrackId | str) -> str: + return track_id.id if isinstance(track_id, TrackId) else track_id + + +def pack(track_id: TrackId | str) -> TrackId: + return track_id if isinstance(track_id, TrackId) else TrackId(track_id) + + class PandasTrackDataset(TrackDataset, PandasDataFrameProvider): @property def track_ids(self) -> frozenset[TrackId]: @@ -366,10 +374,13 @@ def add_all(self, other: Iterable[Track]) -> "PandasTrackDataset": def _add_to_geometry_dataset( self, new_tracks: TrackDataset ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: - updated = dict[RelativeOffsetCoordinate, TrackGeometryDataset]() - for offset, geometries in self._geometry_datasets.items(): - updated[offset] = geometries.add_all(new_tracks) - return updated + # TODO reactivate after remove is implemented in geometry dataset + self._geometry_datasets.clear() + return {} + # updated = dict[RelativeOffsetCoordinate, TrackGeometryDataset]() + # for offset, geometries in self._geometry_datasets.items(): + # updated[offset] = geometries.add_all(new_tracks) + # return updated def __get_tracks(self, other: Iterable[Track]) -> DataFrame: if isinstance(other, PandasDataFrameProvider): @@ -385,7 +396,7 @@ def get_for(self, id: TrackId) -> Optional[Track]: if self._dataset.empty: return None try: - return self._create_track_flyweight(id.id) + return self._create_track_flyweight(unpack(id)) except KeyError: return None @@ -393,14 +404,14 @@ def clear(self) -> "PandasTrackDataset": return PandasTrackDataset(self.track_geometry_factory) def remove(self, track_id: TrackId) -> "PandasTrackDataset": - remaining_tracks = self._dataset.drop(track_id.id, errors="ignore") + remaining_tracks = self._dataset.drop(unpack(track_id), errors="ignore") updated_geometry_datasets = self._remove_from_geometry_dataset([track_id.id]) return PandasTrackDataset.from_dataframe( remaining_tracks, self.track_geometry_factory, updated_geometry_datasets ) def remove_multiple(self, track_ids: set[TrackId]) -> "PandasTrackDataset": - track_ids_primitive = [track_id.id for track_id in track_ids] + track_ids_primitive = [unpack(track_id) for track_id in track_ids] remaining_tracks = self._dataset.drop(track_ids_primitive, errors="ignore") updated_geometry_datasets = self._remove_from_geometry_dataset( track_ids_primitive @@ -412,10 +423,13 @@ def remove_multiple(self, track_ids: set[TrackId]) -> "PandasTrackDataset": def _remove_from_geometry_dataset( self, track_ids: Sequence[str] ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: - updated_dataset = {} - for offset, geometry_dataset in self._geometry_datasets.items(): - updated_dataset[offset] = geometry_dataset.remove(track_ids) - return updated_dataset + # TODO reactivate after remove is implemented in geometry dataset + self._geometry_datasets.clear() + return {} + # updated_dataset = {} + # for offset, geometry_dataset in self._geometry_datasets.items(): + # updated_dataset[offset] = geometry_dataset.remove(track_ids) + # return updated_dataset def as_list(self) -> list[Track]: if (track_ids := self.get_index()) is not None: @@ -468,10 +482,11 @@ def get_index(self) -> Index | None: def _get_geometries_for( self, track_ids: list[str] ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: - geometry_datasets = {} - for offset, geometry_dataset in self._geometry_datasets.items(): - geometry_datasets[offset] = geometry_dataset.get_for(track_ids) - return geometry_datasets + # TODO reactivate after remove is implemented in geometry dataset + return {} + # for offset, geometry_dataset in self._geometry_datasets.items(): + # geometry_datasets[offset] = geometry_dataset.get_for(track_ids) + # return geometry_datasets def __len__(self) -> int: if self._dataset.empty: @@ -602,7 +617,7 @@ def cut_with_section( return self, set() intersection_points = self.intersection_points([section], offset) cut_indices = { - track_id.id: [ + unpack(track_id): [ ip[1].upper_index for ip in sorted(intersection_points, key=lambda ip: ip[1]) ] From b52bc2e687ebec03d72fcd4c7838cc31a972ee40 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 21:03:01 +0200 Subject: [PATCH 021/128] Use pandas geometry dataset --- OTAnalytics/plugin_ui/base_application.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index cf5a4d34c..a79c3a10c 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -185,8 +185,8 @@ TypeCheckingPandasTrackDatasetFactory, ) from OTAnalytics.plugin_datastore.python_track_store import ByMaxConfidence -from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( - ShapelyTrackGeometryDataset, +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + PandasTrackGeometryDataset, ) from OTAnalytics.plugin_datastore.track_store import ( FilterByClassPandasTrackDataset, @@ -996,7 +996,8 @@ def pandas_by_max_confidence(self) -> PandasByMaxConfidence: @cached_property def track_geometry_factory(self) -> TRACK_GEOMETRY_FACTORY: - return ShapelyTrackGeometryDataset.from_track_dataset + return PandasTrackGeometryDataset.from_track_dataset + # return ShapelyTrackGeometryDataset.from_track_dataset def create_format_fixer( From 9cba860ad6f9fceb659ce89d91b05a23d6758468 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 21:26:29 +0200 Subject: [PATCH 022/128] Remove shapely geometry store in base_application --- OTAnalytics/plugin_ui/base_application.py | 1 - 1 file changed, 1 deletion(-) diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index a79c3a10c..be253891e 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -997,7 +997,6 @@ def pandas_by_max_confidence(self) -> PandasByMaxConfidence: @cached_property def track_geometry_factory(self) -> TRACK_GEOMETRY_FACTORY: return PandasTrackGeometryDataset.from_track_dataset - # return ShapelyTrackGeometryDataset.from_track_dataset def create_format_fixer( From a719ad58adb253509a4dfa7bf52247fdd3fd8ef2 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 21:40:55 +0200 Subject: [PATCH 023/128] Implement add_all, remove and get_for --- .../pandas_geometry_store.py | 117 ++++++++++++++++-- OTAnalytics/plugin_datastore/track_store.py | 31 ++--- 2 files changed, 122 insertions(+), 26 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 6d34e85f9..ef314550a 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import Any, Iterable, Optional, Sequence -from pandas import DataFrame +from pandas import DataFrame, concat from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate @@ -649,16 +649,119 @@ def from_track_dataset( ) def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": - # TODO convert to segments and append segments to dataframe - raise NotImplementedError + """Add tracks to existing dataset. + + Pre-existing tracks will be overwritten. + + Args: + tracks (Iterable[Track]): the tracks to add. + + Returns: + TrackGeometryDataset: the dataset with tracks added. + """ + # Convert tracks to DataFrame format + track_data = self._convert_tracks_to_dataframe(tracks) + if track_data.empty: + return self + + # Create segments from track data + new_segments = create_track_segments(track_data) + if new_segments.empty: + return self + + if self.empty: + # If current dataset is empty, return new dataset with the new segments + return PandasTrackGeometryDataset(self._offset, new_segments) + + # Merge with existing segments, overwriting duplicates + # Remove existing segments for tracks that are being added + new_track_ids = new_segments[TRACK_ID].unique() + existing_without_new = self._segments_df[ + ~self._segments_df[TRACK_ID].isin(new_track_ids) + ] + + # Combine existing (without overlaps) and new segments + combined_segments = DataFrame() + if not existing_without_new.empty and not new_segments.empty: + combined_segments = concat( + [existing_without_new, new_segments], ignore_index=True + ) + elif not existing_without_new.empty: + combined_segments = existing_without_new + elif not new_segments.empty: + combined_segments = new_segments + + return PandasTrackGeometryDataset(self._offset, combined_segments) + + def _convert_tracks_to_dataframe(self, tracks: Iterable[Track]) -> DataFrame: + """Convert tracks to DataFrame format. + + Args: + tracks (Iterable[Track]): tracks to convert. + + Returns: + DataFrame: tracks as dataframe with TRACK_ID, OCCURRENCE, X, Y, W, H. + """ + if not tracks: + return DataFrame() + + data = [] + for track in tracks: + track_id = track.id.id + for detection in track.detections: + data.append( + { + TRACK_ID: track_id, + OCCURRENCE: detection.occurrence, + X: detection.x, + Y: detection.y, + W: detection.w, + H: detection.h, + } + ) + + if not data: + return DataFrame() + + return DataFrame(data) def remove(self, ids: Sequence[str]) -> "TrackGeometryDataset": - # TODO remove all segments with given ids - raise NotImplementedError + """Remove track geometries with given ids from dataset. + + Args: + ids (Sequence[str]): the track geometries to remove. + + Returns: + TrackGeometryDataset: the dataset with tracks removed. + """ + if self.empty or not ids: + return self + + # Filter out segments with track IDs in the removal list + remaining_segments = self._segments_df[~self._segments_df[TRACK_ID].isin(ids)] + + return PandasTrackGeometryDataset(self._offset, remaining_segments) def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": - # TODO filter segments by track ids - raise NotImplementedError + """Get geometries for given track ids if they exist. + + Ids that do not exist will not be included in the dataset. + + Args: + track_ids (list[str]): the track ids. + + Returns: + TrackGeometryDataset: the dataset with tracks. + """ + if self.empty or not track_ids: + return PandasTrackGeometryDataset(self._offset) + + # Filter segments to include only those with track IDs in the provided list + filtered_segments = self._segments_df[ + self._segments_df[TRACK_ID].isin(track_ids) + ] + + return PandasTrackGeometryDataset(self._offset, filtered_segments) def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: """Return a set of tracks intersecting a set of sections. diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 765fad863..1006ba71f 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -374,13 +374,10 @@ def add_all(self, other: Iterable[Track]) -> "PandasTrackDataset": def _add_to_geometry_dataset( self, new_tracks: TrackDataset ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: - # TODO reactivate after remove is implemented in geometry dataset - self._geometry_datasets.clear() - return {} - # updated = dict[RelativeOffsetCoordinate, TrackGeometryDataset]() - # for offset, geometries in self._geometry_datasets.items(): - # updated[offset] = geometries.add_all(new_tracks) - # return updated + updated = dict[RelativeOffsetCoordinate, TrackGeometryDataset]() + for offset, geometries in self._geometry_datasets.items(): + updated[offset] = geometries.add_all(new_tracks) + return updated def __get_tracks(self, other: Iterable[Track]) -> DataFrame: if isinstance(other, PandasDataFrameProvider): @@ -423,13 +420,10 @@ def remove_multiple(self, track_ids: set[TrackId]) -> "PandasTrackDataset": def _remove_from_geometry_dataset( self, track_ids: Sequence[str] ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: - # TODO reactivate after remove is implemented in geometry dataset - self._geometry_datasets.clear() - return {} - # updated_dataset = {} - # for offset, geometry_dataset in self._geometry_datasets.items(): - # updated_dataset[offset] = geometry_dataset.remove(track_ids) - # return updated_dataset + updated_dataset = {} + for offset, geometry_dataset in self._geometry_datasets.items(): + updated_dataset[offset] = geometry_dataset.remove(track_ids) + return updated_dataset def as_list(self) -> list[Track]: if (track_ids := self.get_index()) is not None: @@ -482,11 +476,10 @@ def get_index(self) -> Index | None: def _get_geometries_for( self, track_ids: list[str] ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: - # TODO reactivate after remove is implemented in geometry dataset - return {} - # for offset, geometry_dataset in self._geometry_datasets.items(): - # geometry_datasets[offset] = geometry_dataset.get_for(track_ids) - # return geometry_datasets + geometry_datasets = {} + for offset, geometry_dataset in self._geometry_datasets.items(): + geometry_datasets[offset] = geometry_dataset.get_for(track_ids) + return geometry_datasets def __len__(self) -> int: if self._dataset.empty: From 476f2ad474cada0609e89de6862d25202f38a876 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 21:42:47 +0200 Subject: [PATCH 024/128] Fix broken test --- .../track_geometry_store/pandas_geometry_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index ef314550a..2618af8be 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -118,7 +118,7 @@ def create_track_segments(df: DataFrame) -> DataFrame: segments[START_H] = grouped[H].shift(1) # Remove NaN rows and reset index - segments = segments.dropna().reset_index() + segments = segments.dropna().reset_index(drop=True) return segments From 4bacd4fc4bdc8ef26c3880fbab37b448c71d31b4 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 14 Aug 2025 21:44:05 +0200 Subject: [PATCH 025/128] Clean up unused method --- .../track_geometry_store/pandas_geometry_store.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 2618af8be..b1d36b486 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -123,19 +123,6 @@ def create_track_segments(df: DataFrame) -> DataFrame: return segments -def apply_offset(segments_df: DataFrame, offset: RelativeOffsetCoordinate) -> DataFrame: - """Apply an offset to the coordinates of a DataFrame containing track segments.""" - if segments_df.empty: - return segments_df - - segments_df = segments_df.copy() - segments_df[START_X] += segments_df[START_W] * offset.x - segments_df[START_Y] += segments_df[START_H] * offset.y - segments_df[END_X] += segments_df[END_W] * offset.x - segments_df[END_Y] += segments_df[END_H] * offset.y - return segments_df - - def calculate_intersection_parameters( segments_df: DataFrame, line_x1: float, From f027d5d39f68ebaf4902015e6742b008da5cc39e Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 08:53:21 +0200 Subject: [PATCH 026/128] Add script to convert ottrk to feather format --- .pre-commit-config.yaml | 3 +- convert_ottrk_to_feather.py | 188 ++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 convert_ottrk_to_feather.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d65af3f05..d910b407f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,6 +60,7 @@ repos: - pandas-stubs==2.2.3.241126 - pandas==2.2.3 - pre-commit==4.1.0 + - pyarrow==21.0.0 - pytest-asyncio==0.24.0 - pytest-benchmark==5.1.0 - pytest-cov==6.0.0 @@ -97,4 +98,4 @@ repos: - id: shell-fmt-docker args: - -i - - "2" + - '2' diff --git a/convert_ottrk_to_feather.py b/convert_ottrk_to_feather.py new file mode 100644 index 000000000..366950103 --- /dev/null +++ b/convert_ottrk_to_feather.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +Script to convert ottrk files to Apache Arrow/Feather format with metadata export. + +This script reads a single ottrk file using the OttrkParser, extracts the pandas +DataFrame, and saves it in Apache Arrow/Feather format. The metadata is saved as a +separate JSON file. Both output files use the same stem as the input file. + +Requirements: + - pyarrow: Required for feather format export (install with: pip install pyarrow) + +Usage: + python convert_ottrk_to_feather.py + +Example: + python convert_ottrk_to_feather.py data/sample.ottrk + + This will create: + - data/sample.feather (DataFrame in feather format) + - data/sample_metadata.json (metadata in JSON format) +""" + +import argparse +import sys +from pathlib import Path +from typing import Any, Dict + +from OTAnalytics.application.datastore import TrackParseResult +from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY +from OTAnalytics.domain.track_repository import TrackRepository +from OTAnalytics.plugin_datastore.python_track_store import ( + ByMaxConfidence, + PythonTrackDataset, +) +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + PandasTrackGeometryDataset, +) +from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset +from OTAnalytics.plugin_parser.json_parser import write_json +from OTAnalytics.plugin_parser.otvision_parser import ( + DEFAULT_TRACK_LENGTH_LIMIT, + OttrkParser, + PythonDetectionParser, +) + + +def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: + """Create a track geometry factory for PandasTrackDataset.""" + return PandasTrackGeometryDataset.from_track_dataset + + +def create_ottrk_parser() -> OttrkParser: + """Create an OttrkParser with the required detection parser.""" + # Create required dependencies for PythonDetectionParser + track_classification_calculator = ByMaxConfidence() + track_geometry_factory = create_track_geometry_factory() + + # Create empty track repository (no existing tracks when parsing new file) + empty_track_dataset = PythonTrackDataset.from_list([], track_geometry_factory) + track_repository = TrackRepository(empty_track_dataset) + + # Create detection parser with all required dependencies + detection_parser = PythonDetectionParser( + track_classification_calculator=track_classification_calculator, + track_repository=track_repository, + track_geometry_factory=track_geometry_factory, + track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, + ) + return OttrkParser(detection_parser) + + +def convert_to_pandas_dataset( + python_dataset: PythonTrackDataset, track_geometry_factory: TRACK_GEOMETRY_FACTORY +) -> PandasTrackDataset: + """Convert PythonTrackDataset to PandasTrackDataset.""" + tracks_list = python_dataset.as_list() + return PandasTrackDataset.from_list(tracks_list, track_geometry_factory) + + +def create_metadata_dict(parse_result: TrackParseResult) -> Dict[str, Any]: + """Create a metadata dictionary from TrackParseResult.""" + metadata = { + "detection_metadata": { + "detection_classes": list(parse_result.detection_metadata.detection_classes) + }, + "video_metadata": { + "path": parse_result.video_metadata.path, + "recorded_start_date": parse_result.video_metadata.recorded_start_date.isoformat(), # noqa + "recorded_fps": parse_result.video_metadata.recorded_fps, + "number_of_frames": parse_result.video_metadata.number_of_frames, + }, + } + + # Add optional fields if they exist + if parse_result.video_metadata.expected_duration is not None: + metadata["video_metadata"][ + "expected_duration" + ] = parse_result.video_metadata.expected_duration.total_seconds() + + if parse_result.video_metadata.actual_fps is not None: + metadata["video_metadata"][ + "actual_fps" + ] = parse_result.video_metadata.actual_fps + + return metadata + + +def convert_ottrk_to_feather(input_file: Path) -> None: + """ + Convert an ottrk file to Apache Arrow/Feather format with metadata export. + + Args: + input_file: Path to the input ottrk file + """ + if not input_file.exists(): + raise FileNotFoundError(f"Input file not found: {input_file}") + + if not input_file.suffix.lower() == ".ottrk": + raise ValueError(f"Input file must have .ottrk extension: {input_file}") + + # Create output file paths using the same stem as input + output_stem = input_file.stem + output_dir = input_file.parent + feather_file = output_dir / f"{output_stem}.feather" + metadata_file = output_dir / f"{output_stem}_metadata.json" + + print(f"Reading ottrk file: {input_file}") + + # Parse the ottrk file + parser = create_ottrk_parser() + parse_result = parser.parse(input_file) + + print(f"Parsed {len(parse_result.tracks)} tracks") + + # Convert to pandas dataset if needed + if isinstance(parse_result.tracks, PythonTrackDataset): + print("Converting to pandas dataset...") + track_geometry_factory = create_track_geometry_factory() + pandas_dataset = convert_to_pandas_dataset( + parse_result.tracks, track_geometry_factory + ) + elif isinstance(parse_result.tracks, PandasTrackDataset): + pandas_dataset = parse_result.tracks + else: + raise TypeError(f"Unsupported track dataset type: {type(parse_result.tracks)}") + + # Get the pandas DataFrame + df = pandas_dataset.get_data() + print(f"DataFrame shape: {df.shape}") + + # Save DataFrame to feather format + print(f"Saving DataFrame to: {feather_file}") + df.to_feather(feather_file) + + # Create and save metadata + metadata = create_metadata_dict(parse_result) + print(f"Saving metadata to: {metadata_file}") + write_json(metadata, metadata_file) + + print("Conversion completed successfully!") + print("Output files:") + print(f" - Data: {feather_file}") + print(f" - Metadata: {metadata_file}") + + +def main(): + """Main entry point for the script.""" + parser = argparse.ArgumentParser( + description="Convert ottrk files to Apache Arrow/Feather format with metadata " + "export" + ) + parser.add_argument( + "input_file", + type=Path, + help="Path to the input ottrk file", + ) + + args = parser.parse_args() + + try: + convert_ottrk_to_feather(args.input_file) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 835de745a..6c38fabdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ numpy==2.1.1 openpyxl==3.1.5 pandas==2.2.3 pillow==11.1.0 +pyarrow==21.0.0 requests==2.32.3 seaborn==0.13.2 shapely==2.0.7 From 078f2a4f87b1440a70c91251ea6731fd8c7ac034 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 08:58:28 +0200 Subject: [PATCH 027/128] Directly use pandas version --- convert_ottrk_to_feather.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/convert_ottrk_to_feather.py b/convert_ottrk_to_feather.py index 366950103..b8f8a6039 100644 --- a/convert_ottrk_to_feather.py +++ b/convert_ottrk_to_feather.py @@ -27,21 +27,20 @@ from OTAnalytics.application.datastore import TrackParseResult from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY -from OTAnalytics.domain.track_repository import TrackRepository -from OTAnalytics.plugin_datastore.python_track_store import ( - ByMaxConfidence, - PythonTrackDataset, -) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( PandasTrackGeometryDataset, ) -from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset +from OTAnalytics.plugin_datastore.track_store import ( + PandasByMaxConfidence, + PandasTrackDataset, +) from OTAnalytics.plugin_parser.json_parser import write_json from OTAnalytics.plugin_parser.otvision_parser import ( DEFAULT_TRACK_LENGTH_LIMIT, OttrkParser, - PythonDetectionParser, ) +from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: @@ -52,17 +51,12 @@ def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: def create_ottrk_parser() -> OttrkParser: """Create an OttrkParser with the required detection parser.""" # Create required dependencies for PythonDetectionParser - track_classification_calculator = ByMaxConfidence() + track_classification_calculator = PandasByMaxConfidence() track_geometry_factory = create_track_geometry_factory() - # Create empty track repository (no existing tracks when parsing new file) - empty_track_dataset = PythonTrackDataset.from_list([], track_geometry_factory) - track_repository = TrackRepository(empty_track_dataset) - # Create detection parser with all required dependencies - detection_parser = PythonDetectionParser( - track_classification_calculator=track_classification_calculator, - track_repository=track_repository, + detection_parser = PandasDetectionParser( + calculator=track_classification_calculator, track_geometry_factory=track_geometry_factory, track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, ) From 18cc4782042821018e44cbe8b821df0defdab86c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 10:21:12 +0200 Subject: [PATCH 028/128] Implement feather track parser --- OTAnalytics/plugin_parser/feathers_parser.py | 146 +++++++++++ .../plugin_parser/test_feathers_parser.py | 234 ++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 OTAnalytics/plugin_parser/feathers_parser.py create mode 100644 tests/OTAnalytics/plugin_parser/test_feathers_parser.py diff --git a/OTAnalytics/plugin_parser/feathers_parser.py b/OTAnalytics/plugin_parser/feathers_parser.py new file mode 100644 index 000000000..37621d94e --- /dev/null +++ b/OTAnalytics/plugin_parser/feathers_parser.py @@ -0,0 +1,146 @@ +""" +Parser for Apache Arrow/Feather format files with metadata. + +This module provides a TrackParser implementation that reads track data from +feather files and their accompanying metadata JSON files to create TrackParseResult. +""" + +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +import pandas as pd + +from OTAnalytics.application.datastore import ( + DetectionMetadata, + TrackParser, + TrackParseResult, +) +from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY +from OTAnalytics.domain.video import VideoMetadata +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + PandasTrackGeometryDataset, +) +from OTAnalytics.plugin_datastore.track_store import ( + PandasByMaxConfidence, + PandasTrackDataset, +) +from OTAnalytics.plugin_parser.json_parser import parse_json + + +class FeathersParser(TrackParser): + """ + Parse feather files with accompanying metadata JSON files. + + The parser expects two files: + - A .feather file containing the track data as a pandas DataFrame + - A _metadata.json file containing detection and video metadata + + For example, if the input file is "data.feather", the parser will also + look for "data_metadata.json" in the same directory. + """ + + def __init__( + self, + track_geometry_factory: Optional[TRACK_GEOMETRY_FACTORY] = None, + ) -> None: + """ + Initialize the FeathersParser. + + Args: track_geometry_factory: Factory for creating track geometry datasets. + If None, uses PandasTrackGeometryDataset.from_track_dataset. + """ + if track_geometry_factory is None: + track_geometry_factory = PandasTrackGeometryDataset.from_track_dataset + self._track_geometry_factory = track_geometry_factory + + def parse(self, file: Path) -> TrackParseResult: + """ + Parse feather file and its metadata to create TrackParseResult. + + Args: + file: Path to the feather file + + Returns: + TrackParseResult: Contains tracks, detection metadata, and video metadata + + Raises: + FileNotFoundError: If the feather file or metadata file is not found + ValueError: If the file extension is not .feather + """ + if not file.exists(): + raise FileNotFoundError(f"Feather file not found: {file}") + + if not file.suffix.lower() == ".feather": + if file.suffix.lower() == ".ottrk": + file = file.with_suffix(".feather") + else: + raise ValueError(f"Input file must have .feather extension: {file}") + + # Construct metadata file path + metadata_file = file.parent / f"{file.stem}_metadata.json" + if not metadata_file.exists(): + raise FileNotFoundError(f"Metadata file not found: {metadata_file}") + + # Read the feather file + df = pd.read_feather(file) + + # Read the metadata + metadata = parse_json(metadata_file) + + # Create TrackDataset from DataFrame + calculator = PandasByMaxConfidence() + tracks = PandasTrackDataset.from_dataframe( + df, self._track_geometry_factory, calculator=calculator + ) + + # Parse video metadata + video_metadata = self._parse_video_metadata(metadata["video_metadata"]) + + # Parse detection metadata + detection_metadata = self._parse_detection_metadata( + metadata["detection_metadata"] + ) + + return TrackParseResult(tracks, detection_metadata, video_metadata) + + def _parse_video_metadata(self, metadata: dict) -> VideoMetadata: + """ + Parse video metadata from the metadata dictionary. + + Args: + metadata: Dictionary containing video metadata + + Returns: + VideoMetadata: Parsed video metadata object + """ + recorded_start_date = datetime.fromisoformat(metadata["recorded_start_date"]) + + # Parse optional fields + expected_duration = None + if "expected_duration" in metadata: + expected_duration = timedelta(seconds=metadata["expected_duration"]) + + actual_fps = metadata.get("actual_fps") + + return VideoMetadata( + path=metadata["path"], + recorded_start_date=recorded_start_date, + expected_duration=expected_duration, + recorded_fps=metadata["recorded_fps"], + actual_fps=actual_fps, + number_of_frames=metadata["number_of_frames"], + ) + + def _parse_detection_metadata(self, metadata: dict) -> DetectionMetadata: + """ + Parse detection metadata from the metadata dictionary. + + Args: + metadata: Dictionary containing detection metadata + + Returns: + DetectionMetadata: Parsed detection metadata object + """ + detection_classes = frozenset(metadata["detection_classes"]) + return DetectionMetadata(detection_classes) diff --git a/tests/OTAnalytics/plugin_parser/test_feathers_parser.py b/tests/OTAnalytics/plugin_parser/test_feathers_parser.py new file mode 100644 index 000000000..f7ce392a9 --- /dev/null +++ b/tests/OTAnalytics/plugin_parser/test_feathers_parser.py @@ -0,0 +1,234 @@ +""" +Tests for the FeathersParser module. +""" + +import tempfile +from datetime import datetime, timedelta +from pathlib import Path +from unittest import TestCase +from unittest.mock import Mock, patch + +import pandas as pd +import pytest + +from OTAnalytics.application.datastore import DetectionMetadata, TrackParseResult +from OTAnalytics.domain.video import VideoMetadata +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + PandasTrackGeometryDataset, +) +from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset +from OTAnalytics.plugin_parser.feathers_parser import FeathersParser + + +class TestFeathersParser(TestCase): + + def setUp(self) -> None: + """Set up test fixtures.""" + self.parser = FeathersParser() + + # Sample DataFrame for testing + self.sample_df = pd.DataFrame( + { + "track_id": ["1", "1", "2", "2"], + "frame": [1, 2, 1, 2], + "x": [100.0, 105.0, 200.0, 205.0], + "y": [100.0, 105.0, 200.0, 205.0], + "w": [50.0, 50.0, 60.0, 60.0], + "h": [80.0, 80.0, 90.0, 90.0], + "classification": ["car", "car", "truck", "truck"], + "confidence": [0.9, 0.85, 0.92, 0.88], + "occurrence": [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + ], + "video_name": [ + "test_video.mp4", + "test_video.mp4", + "test_video.mp4", + "test_video.mp4", + ], + "input_file": ["test.ottrk", "test.ottrk", "test.ottrk", "test.ottrk"], + "interpolated_detection": [False, False, False, False], + "original_track_id": ["1", "1", "2", "2"], + } + ) + + # Sample metadata for testing + self.sample_metadata = { + "detection_metadata": {"detection_classes": ["car", "truck", "bus"]}, + "video_metadata": { + "path": "test_video.mp4", + "recorded_start_date": "2023-01-01T10:00:00+00:00", + "recorded_fps": 30.0, + "number_of_frames": 900, + "expected_duration": 30.0, + "actual_fps": 29.97, + }, + } + + def test_init_default_geometry_factory(self) -> None: + """Test initialization with default geometry factory.""" + parser = FeathersParser() + self.assertEqual( + parser._track_geometry_factory, + PandasTrackGeometryDataset.from_track_dataset, + ) + + def test_init_custom_geometry_factory(self) -> None: + """Test initialization with custom geometry factory.""" + custom_factory = Mock() + parser = FeathersParser(track_geometry_factory=custom_factory) + self.assertEqual(parser._track_geometry_factory, custom_factory) + + def test_parse_file_not_found(self) -> None: + """Test parsing when feather file doesn't exist.""" + non_existent_file = Path("/non/existent/file.feather") + + with self.assertRaises(FileNotFoundError) as context: + self.parser.parse(non_existent_file) + + self.assertIn("Feather file not found", str(context.exception)) + + def test_parse_wrong_extension(self) -> None: + """Test parsing file with wrong extension.""" + with tempfile.NamedTemporaryFile(suffix=".txt") as temp_file: + temp_path = Path(temp_file.name) + + with self.assertRaises(ValueError) as context: + self.parser.parse(temp_path) + + self.assertIn("must have .feather extension", str(context.exception)) + + def test_parse_metadata_file_not_found(self) -> None: + """Test parsing when metadata file doesn't exist.""" + with tempfile.NamedTemporaryFile(suffix=".feather", delete=False) as temp_file: + temp_path = Path(temp_file.name) + # Save empty DataFrame to create valid feather file + pd.DataFrame().to_feather(temp_file.name) + + try: + with self.assertRaises(FileNotFoundError) as context: + self.parser.parse(temp_path) + + self.assertIn("Metadata file not found", str(context.exception)) + finally: + temp_path.unlink() # Clean up + + @patch("OTAnalytics.plugin_parser.feathers_parser.parse_json") + @patch("pandas.read_feather") + @patch("OTAnalytics.plugin_datastore.track_store.PandasTrackDataset.from_dataframe") + def test_parse_success( + self, mock_from_dataframe: Mock, mock_read_feather: Mock, mock_parse_json: Mock + ) -> None: + """Test successful parsing of feather file and metadata.""" + # Set up mocks + mock_read_feather.return_value = self.sample_df + mock_parse_json.return_value = self.sample_metadata + mock_track_dataset = Mock(spec=PandasTrackDataset) + mock_from_dataframe.return_value = mock_track_dataset + + with tempfile.NamedTemporaryFile( + suffix=".feather", delete=False + ) as feather_file: + feather_path = Path(feather_file.name) + # Create the metadata file + metadata_path = feather_path.parent / f"{feather_path.stem}_metadata.json" + metadata_path.write_text('{"test": "data"}') + + try: + result = self.parser.parse(feather_path) + + # Verify result type and structure + self.assertIsInstance(result, TrackParseResult) + self.assertEqual(result.tracks, mock_track_dataset) + + # Verify video metadata + video_metadata = result.video_metadata + self.assertIsInstance(video_metadata, VideoMetadata) + self.assertEqual(video_metadata.path, "test_video.mp4") + self.assertEqual(video_metadata.recorded_fps, 30.0) + self.assertEqual(video_metadata.number_of_frames, 900) + self.assertEqual(video_metadata.actual_fps, 29.97) + self.assertEqual(video_metadata.expected_duration, timedelta(seconds=30.0)) + + # Verify detection metadata + detection_metadata = result.detection_metadata + self.assertIsInstance(detection_metadata, DetectionMetadata) + expected_classes = frozenset(["car", "truck", "bus"]) + self.assertEqual(detection_metadata.detection_classes, expected_classes) + + # Verify mock calls + mock_read_feather.assert_called_once_with(feather_path) + mock_parse_json.assert_called_once_with(metadata_path) + mock_from_dataframe.assert_called_once() + + finally: + # Clean up + feather_path.unlink() + if metadata_path.exists(): + metadata_path.unlink() + + def test_parse_video_metadata_minimal(self) -> None: + """Test parsing video metadata with minimal required fields.""" + minimal_metadata = { + "path": "video.mp4", + "recorded_start_date": "2023-01-01T10:00:00+00:00", + "recorded_fps": 30.0, + "number_of_frames": 900, + } + + result = self.parser._parse_video_metadata(minimal_metadata) + + self.assertIsInstance(result, VideoMetadata) + self.assertEqual(result.path, "video.mp4") + self.assertEqual(result.recorded_fps, 30.0) + self.assertEqual(result.number_of_frames, 900) + self.assertIsNone(result.expected_duration) + self.assertIsNone(result.actual_fps) + + def test_parse_video_metadata_with_optional_fields(self) -> None: + """Test parsing video metadata with all optional fields.""" + full_metadata = { + "path": "video.mp4", + "recorded_start_date": "2023-01-01T10:00:00+00:00", + "recorded_fps": 30.0, + "number_of_frames": 900, + "expected_duration": 30.0, + "actual_fps": 29.97, + } + + result = self.parser._parse_video_metadata(full_metadata) + + self.assertIsInstance(result, VideoMetadata) + self.assertEqual(result.path, "video.mp4") + self.assertEqual(result.recorded_fps, 30.0) + self.assertEqual(result.number_of_frames, 900) + self.assertEqual(result.expected_duration, timedelta(seconds=30.0)) + self.assertEqual(result.actual_fps, 29.97) + + def test_parse_detection_metadata(self) -> None: + """Test parsing detection metadata.""" + detection_metadata = { + "detection_classes": ["car", "truck", "bus", "motorcycle"] + } + + result = self.parser._parse_detection_metadata(detection_metadata) + + self.assertIsInstance(result, DetectionMetadata) + expected_classes = frozenset(["car", "truck", "bus", "motorcycle"]) + self.assertEqual(result.detection_classes, expected_classes) + + def test_parse_detection_metadata_empty_classes(self) -> None: + """Test parsing detection metadata with empty classes list.""" + detection_metadata: dict[str, list] = {"detection_classes": []} + + result = self.parser._parse_detection_metadata(detection_metadata) + + self.assertIsInstance(result, DetectionMetadata) + self.assertEqual(result.detection_classes, frozenset()) + + +if __name__ == "__main__": + pytest.main([__file__]) From fb0cf9e59fbf9885aefadfea3d5aa5f00b977263 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 10:27:20 +0200 Subject: [PATCH 029/128] Use feathers parser --- OTAnalytics/plugin_ui/base_application.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index be253891e..bd16b108a 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -223,6 +223,7 @@ FillZerosExporterFactory, SimpleExporterFactory, ) +from OTAnalytics.plugin_parser.feathers_parser import FeathersParser from OTAnalytics.plugin_parser.json_parser import parse_json from OTAnalytics.plugin_parser.otconfig_parser import ( FixMissingAnalysis, @@ -604,6 +605,9 @@ def track_repository(self) -> TrackRepository: ) def _create_track_parser(self) -> TrackParser: + return self._create_feathers_parser() + + def _create_ottrk_parser(self) -> TrackParser: detection_parser = PandasDetectionParser( self.pandas_by_max_confidence, self.track_geometry_factory, @@ -611,6 +615,9 @@ def _create_track_parser(self) -> TrackParser: ) return OttrkParser(detection_parser) + def _create_feathers_parser(self) -> TrackParser: + return FeathersParser() + def _create_stream_track_parser(self) -> StreamTrackParser: return StreamOttrkParser( detection_parser=PythonStreamDetectionParser( From 0793c7336752b6d076894fa1bd183cbb2f8805fb Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 10:27:58 +0200 Subject: [PATCH 030/128] Convert a bunch of files to feathers --- convert_ottrk_to_feather.py | 48 +++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/convert_ottrk_to_feather.py b/convert_ottrk_to_feather.py index b8f8a6039..945baeb55 100644 --- a/convert_ottrk_to_feather.py +++ b/convert_ottrk_to_feather.py @@ -2,7 +2,7 @@ """ Script to convert ottrk files to Apache Arrow/Feather format with metadata export. -This script reads a single ottrk file using the OttrkParser, extracts the pandas +This script reads one or more ottrk files using the OttrkParser, extracts the pandas DataFrame, and saves it in Apache Arrow/Feather format. The metadata is saved as a separate JSON file. Both output files use the same stem as the input file. @@ -10,14 +10,21 @@ - pyarrow: Required for feather format export (install with: pip install pyarrow) Usage: - python convert_ottrk_to_feather.py + python convert_ottrk_to_feather.py [input_file2.ottrk ...] -Example: +Examples: + # Convert single file python convert_ottrk_to_feather.py data/sample.ottrk - This will create: - - data/sample.feather (DataFrame in feather format) - - data/sample_metadata.json (metadata in JSON format) + # Convert multiple files python convert_ottrk_to_feather.py data/sample1.ottrk + data/sample2.ottrk data/sample3.ottrk + + # Convert all ottrk files in a directory using shell globbing + python convert_ottrk_to_feather.py data/*.ottrk + + Each input file will create: + - .feather (DataFrame in feather format) + - _metadata.json (metadata in JSON format) """ import argparse @@ -164,17 +171,34 @@ def main(): "export" ) parser.add_argument( - "input_file", + "input_files", type=Path, - help="Path to the input ottrk file", + nargs="+", + help="Path(s) to the input ottrk file(s)", ) args = parser.parse_args() - try: - convert_ottrk_to_feather(args.input_file) - except Exception as e: - print(f"Error: {e}", file=sys.stderr) + # Process each input file + failed_files = [] + for input_file in args.input_files: + try: + convert_ottrk_to_feather(input_file) + except Exception as e: + print(f"Error processing {input_file}: {e}", file=sys.stderr) + failed_files.append(input_file) + + # Report overall results + total_files = len(args.input_files) + successful_files = total_files - len(failed_files) + + print("\nProcessing complete:") + print(f" - Successfully converted: {successful_files}/{total_files} files") + + if failed_files: + print(f" - Failed to convert: {len(failed_files)} files") + for failed_file in failed_files: + print(f" * {failed_file}") sys.exit(1) From 88d296a4e4bed354f2dea9ee0ddca3152ead2514 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 12:25:42 +0200 Subject: [PATCH 031/128] Allow TrackParser implementations to parse multiple files more optimized --- OTAnalytics/application/datastore.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/OTAnalytics/application/datastore.py b/OTAnalytics/application/datastore.py index 17357e683..d63c9ece5 100644 --- a/OTAnalytics/application/datastore.py +++ b/OTAnalytics/application/datastore.py @@ -53,7 +53,32 @@ class TrackParseResult: video_metadata: VideoMetadata +@dataclass(frozen=True) +class TracksParseResult: + tracks: TrackDataset + detections_metadata: list[DetectionMetadata] + videos_metadata: list[VideoMetadata] + + +def combine_track_datasets(results: list[TrackParseResult]) -> TrackDataset: + if not results: + raise ValueError("No results to combine") + tracks = results[0].tracks + for result in results[1:]: + tracks.add_all(result.tracks) + return tracks + + class TrackParser(ABC): + def parse_files(self, files: list[Path]) -> TracksParseResult: + if not files: + raise ValueError("No files to parse") + results = [self.parse(file) for file in files] + tracks = combine_track_datasets(results) + detections_metadata = [result.detection_metadata for result in results] + videos_metadata = [result.video_metadata for result in results] + return TracksParseResult(tracks, detections_metadata, videos_metadata) + @abstractmethod def parse(self, file: Path) -> TrackParseResult: raise NotImplementedError From 697e6005e6f665fb730ccee359523f87bdb0d135 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 12:27:43 +0200 Subject: [PATCH 032/128] Parse multiple files at once in FeathersParser --- OTAnalytics/plugin_parser/feathers_parser.py | 79 +++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_parser/feathers_parser.py b/OTAnalytics/plugin_parser/feathers_parser.py index 37621d94e..75ca32e44 100644 --- a/OTAnalytics/plugin_parser/feathers_parser.py +++ b/OTAnalytics/plugin_parser/feathers_parser.py @@ -15,7 +15,9 @@ DetectionMetadata, TrackParser, TrackParseResult, + TracksParseResult, ) +from OTAnalytics.application.logger import logger from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY from OTAnalytics.domain.video import VideoMetadata from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( @@ -28,6 +30,25 @@ from OTAnalytics.plugin_parser.json_parser import parse_json +def use_feathers_files(files: list[Path]) -> list[Path]: + raised_exceptions: list[Exception] = [] + result = [] + for file in files: + try: + if not file.suffix.lower() == ".feather": + if file.suffix.lower() == ".ottrk": + result.append(file.with_suffix(".feather")) + else: + raise ValueError(f"Input file must have .feather extension: {file}") + except Exception as cause: + raised_exceptions.append(cause) + if raised_exceptions: + raise ExceptionGroup( + "Errors occurred while loading the track files:", raised_exceptions + ) + return result + + class FeathersParser(TrackParser): """ Parse feather files with accompanying metadata JSON files. @@ -54,6 +75,60 @@ def __init__( track_geometry_factory = PandasTrackGeometryDataset.from_track_dataset self._track_geometry_factory = track_geometry_factory + def parse_files(self, files: list[Path]) -> TracksParseResult: + """ + Parse feather file and its metadata to create TrackParseResult. + + Args: + file: Path to the feather file + + Returns: + TrackParseResult: Contains tracks, detection metadata, and video metadata + + Raises: + FileNotFoundError: If the feather file or metadata file is not found + ValueError: If the file extension is not .feather + """ + logger().info(f"Parsing {len(files)} track files...") + files_to_process = use_feathers_files(files) + videos_metadata = [] + detections_metadata = [] + data_frames = [] + for file in files_to_process: + if not file.exists(): + raise FileNotFoundError(f"Feather file not found: {file}") + # Construct metadata file path + metadata_file = file.parent / f"{file.stem}_metadata.json" + if not metadata_file.exists(): + raise FileNotFoundError(f"Metadata file not found: {metadata_file}") + + # Read the feather file + df = pd.read_feather(file) + data_frames.append(df) + + # Read the metadata + metadata = parse_json(metadata_file) + + # Parse video metadata + video_metadata = self._parse_video_metadata(metadata["video_metadata"]) + videos_metadata.append(video_metadata) + + # Parse detection metadata + detection_metadata = self._parse_detection_metadata( + metadata["detection_metadata"] + ) + detections_metadata.append(detection_metadata) + logger().info(f"{len(files)} track files parsed.") + + df = pd.concat(data_frames) + # Create TrackDataset from DataFrame + calculator = PandasByMaxConfidence() + tracks = PandasTrackDataset.from_dataframe( + df, self._track_geometry_factory, calculator=calculator + ) + logger().info("TrackDataset created.") + return TracksParseResult(tracks, detections_metadata, videos_metadata) + def parse(self, file: Path) -> TrackParseResult: """ Parse feather file and its metadata to create TrackParseResult. @@ -68,8 +143,6 @@ def parse(self, file: Path) -> TrackParseResult: FileNotFoundError: If the feather file or metadata file is not found ValueError: If the file extension is not .feather """ - if not file.exists(): - raise FileNotFoundError(f"Feather file not found: {file}") if not file.suffix.lower() == ".feather": if file.suffix.lower() == ".ottrk": @@ -77,6 +150,8 @@ def parse(self, file: Path) -> TrackParseResult: else: raise ValueError(f"Input file must have .feather extension: {file}") + if not file.exists(): + raise FileNotFoundError(f"Feather file not found: {file}") # Construct metadata file path metadata_file = file.parent / f"{file.stem}_metadata.json" if not metadata_file.exists(): From 3b6ece16ab6447eb11a4522bac84a4d261ccbd89 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 12:35:10 +0200 Subject: [PATCH 033/128] Use multi file parsing when loading ottrk files --- .../application/use_cases/load_track_files.py | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/application/use_cases/load_track_files.py b/OTAnalytics/application/use_cases/load_track_files.py index 34113d1ea..0922f0e5b 100644 --- a/OTAnalytics/application/use_cases/load_track_files.py +++ b/OTAnalytics/application/use_cases/load_track_files.py @@ -4,12 +4,18 @@ TrackParser, TrackToVideoRepository, TrackVideoParser, + VideoParser, ) from OTAnalytics.application.logger import logger from OTAnalytics.application.state import TracksMetadata, VideosMetadata from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository +from OTAnalytics.plugin_parser.otvision_parser import ( + CachedVideoParser, + SimpleVideoParser, +) +from OTAnalytics.plugin_video_processing.video_reader import PyAvVideoReader class LoadTrackFiles: @@ -42,6 +48,46 @@ def __call__(self, files: list[Path]) -> None: Args: files (Path): files in ottrk format. """ + self._load_all_at_once(files) + + def _load_all_at_once(self, files: list[Path]) -> None: + if not files: + return + parent_folder = files[0].parent + files_to_load = [ + file for file in files if not self._is_file_already_loaded(file) + ] + self._log_already_loaded_files(files, files_to_load) + if not files_to_load: + return + logger().info(f"Loading {len(files_to_load)} track files and videos...") + parse_result = self._track_parser.parse_files(files_to_load) + for video_metadata in parse_result.videos_metadata: + self._videos_metadata.update(video_metadata) + + videos = [ + self._video_parser.parse( + parent_folder / video_metadata.path, video_metadata + ) + for video_metadata in parse_result.videos_metadata + ] + self._video_repository.add_all(videos) + # self._track_to_video_repository.add_all(track_ids, videos) + self._track_repository.add_all(parse_result.tracks) + self._track_file_repository.add_all(files_to_load) + for detection_metadata in parse_result.detections_metadata: + self._tracks_metadata.update_detection_classes( + detection_metadata.detection_classes + ) + logger().info(f"Loaded {len(files_to_load)} track files and videos...") + + @property + def _video_parser(self) -> VideoParser: + return CachedVideoParser( + SimpleVideoParser(PyAvVideoReader(self._videos_metadata)) + ) + + def _load_one_by_one(self, files: list[Path]) -> None: raised_exceptions: list[Exception] = [] for file in self._progressbar( files, unit="files", description="Processed ottrk files: " @@ -70,15 +116,25 @@ def load(self, file: Path) -> None: file (Path): file in ottrk format """ parse_result = self._track_parser.parse(file) - track_ids = list(parse_result.tracks.track_ids) - track_ids, videos = self._track_video_parser.parse( - file, track_ids, parse_result.video_metadata + # track_ids = list(parse_result.tracks.track_ids) + # track_ids, videos = self._track_video_parser.parse( + # file, track_ids, parse_result.video_metadata + # ) + video = self._video_parser.parse( + file.parent / parse_result.video_metadata.path, parse_result.video_metadata ) self._videos_metadata.update(parse_result.video_metadata) - self._video_repository.add_all(videos) - self._track_to_video_repository.add_all(track_ids, videos) + self._video_repository.add(video) + # self._track_to_video_repository.add_all(track_ids, videos) self._track_repository.add_all(parse_result.tracks) self._track_file_repository.add(file) self._tracks_metadata.update_detection_classes( parse_result.detection_metadata.detection_classes ) + + def _log_already_loaded_files( + self, files: list[Path], files_to_load: list[Path] + ) -> None: + already_loaded_files = set(files) - set(files_to_load) + for file in already_loaded_files: + logger().warning(f"File '{file}' already loaded. Skipping... ") From 0b1a7f678a04ce56038162b1e21add056fc6fae5 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 14:39:13 +0200 Subject: [PATCH 034/128] Remove unnecessary code --- OTAnalytics/application/application.py | 14 --- OTAnalytics/application/datastore.py | 103 +----------------- OTAnalytics/application/state.py | 4 +- .../use_cases/clear_repositories.py | 6 - .../application/use_cases/load_track_files.py | 30 +---- .../use_cases/track_to_video_repository.py | 16 --- OTAnalytics/plugin_parser/otvision_parser.py | 23 +--- OTAnalytics/plugin_ui/base_application.py | 21 +--- .../OTAnalytics/application/test_datastore.py | 22 ---- .../use_cases/test_clear_repositories.py | 6 - .../use_cases/test_load_track_files.py | 10 +- .../use_cases/track_to_video_repository.py | 14 --- 12 files changed, 14 insertions(+), 255 deletions(-) delete mode 100644 OTAnalytics/application/use_cases/track_to_video_repository.py delete mode 100644 tests/OTAnalytics/application/use_cases/track_to_video_repository.py diff --git a/OTAnalytics/application/application.py b/OTAnalytics/application/application.py index dfba0ca03..a671461ad 100644 --- a/OTAnalytics/application/application.py +++ b/OTAnalytics/application/application.py @@ -82,7 +82,6 @@ SectionId, SectionListObserver, ) -from OTAnalytics.domain.track import TrackId, TrackImage from OTAnalytics.domain.types import EventType from OTAnalytics.domain.video import Video, VideoListObserver @@ -412,19 +411,6 @@ def save_otflow(self, file: Path) -> None: """ self._save_otflow.save(file) - def get_image_of_track(self, track_id: TrackId) -> Optional[TrackImage]: - """ - Retrieve an image for the given track. - - Args: - track_id (TrackId): identifier for the track - - Returns: - Optional[TrackImage]: an image of the track if the track is available and - the image can be loaded - """ - return self._datastore.get_image_of_track(track_id) - def create_events(self) -> None: """ Intersect all tracks with all sections and write the events into the event diff --git a/OTAnalytics/application/datastore.py b/OTAnalytics/application/datastore.py index d63c9ece5..f42d22a6e 100644 --- a/OTAnalytics/application/datastore.py +++ b/OTAnalytics/application/datastore.py @@ -1,7 +1,8 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from datetime import datetime from pathlib import Path -from typing import Iterable, Optional, Sequence, Tuple +from typing import Iterable, Optional, Sequence from OTAnalytics.application.export_formats.export_mode import OVERWRITE from OTAnalytics.application.project import Project @@ -26,7 +27,6 @@ SectionListObserver, SectionRepository, ) -from OTAnalytics.domain.track import TrackId, TrackImage from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset from OTAnalytics.domain.track_repository import ( TrackFileRepository, @@ -114,77 +114,6 @@ def convert( pass -class TrackToVideoRepository: - """ - Repository containing the videos per track. - """ - - def __init__(self) -> None: - self._videos: dict[TrackId, Video] = {} - - def add(self, track_id: TrackId, video: Video) -> None: - """ - Add a video for a track id. - - Args: - track_id (TrackId): id of the track - video (Video): video containing the track - """ - self._videos[track_id] = video - - def add_all(self, track_ids: Iterable[TrackId], videos: Iterable[Video]) -> None: - """ - Add all videos for all tracks. - - Args: - track_ids (Iterable[TrackId]): track ids to be added - videos (Iterable[Video]): videos per track id to be added - """ - for track_id, video in zip(track_ids, videos): - self.add(track_id, video) - - def get_video_for(self, track_id: TrackId) -> Optional[Video]: - """ - Retrieve a video for the given track id. - - Args: - track_id (TrackId): id of the track to get a video for - - Returns: - Optional[Video]: video of the track if a video exists for the track - """ - return self._videos.get(track_id) - - def clear(self) -> None: - """ - Clear the repository. - """ - self._videos.clear() - - -class TrackVideoParser(ABC): - """ - Parse the information about videos from a track file - """ - - @abstractmethod - def parse( - self, file: Path, track_ids: list[TrackId], metadata: VideoMetadata - ) -> Tuple[list[TrackId], list[Video]]: - """ - Parse the given file in ottrk format and retrieve video information from it - - Args: - file (Path): file in ottrk format - track_ids (list[TrackId]): track ids to get videos for - metadata (VideoMetadata): the video metadata - - Returns: - Tuple[list[TrackId], list[Video]]: track ids and the corresponding videos - """ - pass - - class Datastore: """ Central element to hold data in the application. @@ -199,17 +128,14 @@ def __init__( flow_repository: FlowRepository, event_repository: EventRepository, event_list_parser: EventListParser, - track_to_video_repository: TrackToVideoRepository, video_repository: VideoRepository, video_parser: VideoParser, - track_video_parser: TrackVideoParser, progressbar: ProgressbarBuilder, remark_repository: RemarkRepository, ) -> None: self._track_parser = track_parser self._event_list_parser = event_list_parser self._video_parser = video_parser - self._track_video_parser = track_video_parser self._track_repository = track_repository self._track_file_repository = track_file_repository self._section_repository = section_repository @@ -217,7 +143,6 @@ def __init__( self._event_repository = event_repository self._video_repository = video_repository self._remark_repository = remark_repository - self._track_to_video_repository = track_to_video_repository self._progressbar = progressbar self.project = Project(name="", start_date=None, metadata=None) @@ -246,7 +171,6 @@ def clear_repositories(self) -> None: self._event_repository.clear() self._section_repository.clear() self._flow_repository.clear() - self._track_to_video_repository.clear() self._track_repository.clear() self._video_repository.clear() @@ -455,27 +379,8 @@ def set_section_plugin_data(self, section_id: SectionId, plugin_data: dict) -> N def get_video_at(self, file: Path) -> Optional[Video]: return self._video_repository.get(file) - def get_video_for(self, track_id: TrackId) -> Optional[Video]: - return self._track_to_video_repository.get_video_for(track_id) + def get_video_by_date(self, current: datetime) -> list[Video]: + return self._video_repository.get_by_date(current) def get_all_videos(self) -> list[Video]: return self._video_repository.get_all() - - def get_image_of_track( - self, - track_id: TrackId, - frame: int = 0, - ) -> Optional[TrackImage]: - """ - Retrieve an image for the given track. - - Args: - track_id (TrackId): identifier for the track - - Returns: - Optional[TrackImage]: an image of the track if the track is available and - the image can be loaded - """ - return ( - video.get_frame(frame) if (video := self.get_video_for(track_id)) else None - ) diff --git a/OTAnalytics/application/state.py b/OTAnalytics/application/state.py index f806a5db8..17b8138ea 100644 --- a/OTAnalytics/application/state.py +++ b/OTAnalytics/application/state.py @@ -360,8 +360,8 @@ def notify_tracks(self, track_event: TrackRepositoryEvent) -> None: all_tracks = self._datastore.get_all_tracks() if track_event.added: first_track = next(iter(all_tracks)) - if video := self._datastore.get_video_for(first_track.id): - self._track_view_state.selected_videos.set([video]) + if video := self._datastore.get_video_by_date(first_track.start): + self._track_view_state.selected_videos.set(video) def notify_videos(self, videos: list[Video]) -> None: if videos: diff --git a/OTAnalytics/application/use_cases/clear_repositories.py b/OTAnalytics/application/use_cases/clear_repositories.py index e3fdd9a1d..946a10018 100644 --- a/OTAnalytics/application/use_cases/clear_repositories.py +++ b/OTAnalytics/application/use_cases/clear_repositories.py @@ -5,9 +5,6 @@ ) from OTAnalytics.application.use_cases.section_repository import ClearAllSections from OTAnalytics.application.use_cases.track_repository import ClearAllTracks -from OTAnalytics.application.use_cases.track_to_video_repository import ( - ClearAllTrackToVideos, -) from OTAnalytics.application.use_cases.video_repository import ClearAllVideos @@ -30,7 +27,6 @@ def __init__( clear_all_flows: ClearAllFlows, clear_all_intersections: ClearAllIntersections, clear_all_sections: ClearAllSections, - clear_all_track_to_videos: ClearAllTrackToVideos, clear_all_tracks: ClearAllTracks, clear_all_videos: ClearAllVideos, ): @@ -38,7 +34,6 @@ def __init__( self._clear_all_flows = clear_all_flows self._clear_all_intersections = clear_all_intersections self._clear_all_sections = clear_all_sections - self._clear_all_track_to_videos = clear_all_track_to_videos self._clear_all_tracks = clear_all_tracks self._clear_all_videos = clear_all_videos @@ -48,6 +43,5 @@ def __call__(self) -> None: self._clear_all_flows() self._clear_all_intersections.clear() self._clear_all_sections() - self._clear_all_track_to_videos() self._clear_all_tracks() self._clear_all_videos() diff --git a/OTAnalytics/application/use_cases/load_track_files.py b/OTAnalytics/application/use_cases/load_track_files.py index 0922f0e5b..4a62efc6d 100644 --- a/OTAnalytics/application/use_cases/load_track_files.py +++ b/OTAnalytics/application/use_cases/load_track_files.py @@ -1,42 +1,30 @@ from pathlib import Path -from OTAnalytics.application.datastore import ( - TrackParser, - TrackToVideoRepository, - TrackVideoParser, - VideoParser, -) +from OTAnalytics.application.datastore import TrackParser, VideoParser from OTAnalytics.application.logger import logger from OTAnalytics.application.state import TracksMetadata, VideosMetadata from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository -from OTAnalytics.plugin_parser.otvision_parser import ( - CachedVideoParser, - SimpleVideoParser, -) -from OTAnalytics.plugin_video_processing.video_reader import PyAvVideoReader class LoadTrackFiles: def __init__( self, track_parser: TrackParser, - track_video_parser: TrackVideoParser, track_repository: TrackRepository, track_file_repository: TrackFileRepository, video_repository: VideoRepository, - track_to_video_repository: TrackToVideoRepository, + video_parser: VideoParser, progressbar: ProgressbarBuilder, tracks_metadata: TracksMetadata, videos_metadata: VideosMetadata, ) -> None: self._track_parser = track_parser - self._track_video_parser = track_video_parser self._track_repository = track_repository self._track_file_repository = track_file_repository self._video_repository = video_repository - self._track_to_video_repository = track_to_video_repository + self._video_parser = video_parser self._progressbar = progressbar self._tracks_metadata = tracks_metadata self._videos_metadata = videos_metadata @@ -72,7 +60,6 @@ def _load_all_at_once(self, files: list[Path]) -> None: for video_metadata in parse_result.videos_metadata ] self._video_repository.add_all(videos) - # self._track_to_video_repository.add_all(track_ids, videos) self._track_repository.add_all(parse_result.tracks) self._track_file_repository.add_all(files_to_load) for detection_metadata in parse_result.detections_metadata: @@ -81,12 +68,6 @@ def _load_all_at_once(self, files: list[Path]) -> None: ) logger().info(f"Loaded {len(files_to_load)} track files and videos...") - @property - def _video_parser(self) -> VideoParser: - return CachedVideoParser( - SimpleVideoParser(PyAvVideoReader(self._videos_metadata)) - ) - def _load_one_by_one(self, files: list[Path]) -> None: raised_exceptions: list[Exception] = [] for file in self._progressbar( @@ -116,16 +97,11 @@ def load(self, file: Path) -> None: file (Path): file in ottrk format """ parse_result = self._track_parser.parse(file) - # track_ids = list(parse_result.tracks.track_ids) - # track_ids, videos = self._track_video_parser.parse( - # file, track_ids, parse_result.video_metadata - # ) video = self._video_parser.parse( file.parent / parse_result.video_metadata.path, parse_result.video_metadata ) self._videos_metadata.update(parse_result.video_metadata) self._video_repository.add(video) - # self._track_to_video_repository.add_all(track_ids, videos) self._track_repository.add_all(parse_result.tracks) self._track_file_repository.add(file) self._tracks_metadata.update_detection_classes( diff --git a/OTAnalytics/application/use_cases/track_to_video_repository.py b/OTAnalytics/application/use_cases/track_to_video_repository.py deleted file mode 100644 index 4efdaef8b..000000000 --- a/OTAnalytics/application/use_cases/track_to_video_repository.py +++ /dev/null @@ -1,16 +0,0 @@ -from OTAnalytics.application.datastore import TrackToVideoRepository - - -class ClearAllTrackToVideos: - """Clear the track to video repository. - - Args: - track_to_video_repository (TrackToVideoRepository): the repository to clear. - """ - - def __init__(self, track_to_video_repository: TrackToVideoRepository) -> None: - self._track_to_video_repository = track_to_video_repository - - def __call__(self) -> None: - """Clear the track to video repository.""" - self._track_to_video_repository.clear() diff --git a/OTAnalytics/plugin_parser/otvision_parser.py b/OTAnalytics/plugin_parser/otvision_parser.py index 53700a62c..820a6f492 100644 --- a/OTAnalytics/plugin_parser/otvision_parser.py +++ b/OTAnalytics/plugin_parser/otvision_parser.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Callable, Iterable, Optional, Sequence, Tuple +from typing import Any, Callable, Iterable, Optional, Sequence import OTAnalytics.plugin_parser.ottrk_dataformat as ottrk_format from OTAnalytics import version @@ -15,7 +15,6 @@ EventListParser, TrackParser, TrackParseResult, - TrackVideoParser, VideoParser, ) from OTAnalytics.application.logger import logger @@ -1020,26 +1019,6 @@ def convert( return self._other.convert(videos=videos, relative_to=relative_to) -class OttrkVideoParser(TrackVideoParser): - def __init__( - self, - video_parser: VideoParser, - track_format_fixer: OttrkFormatFixer = OttrkFormatFixer(), - ) -> None: - self._video_parser = video_parser - self._track_format_fixer = track_format_fixer - - def parse( - self, file: Path, track_ids: list[TrackId], metadata: VideoMetadata - ) -> Tuple[list[TrackId], list[Video]]: - video = self._video_parser.parse(file.parent / metadata.path, metadata) - return track_ids, [video] * len(track_ids) - - def __parse_recorded_start_date(self, metadata: dict) -> datetime: - start_date = float(metadata[ottrk_format.RECORDED_START_DATE]) - return datetime.fromtimestamp(start_date, tz=timezone.utc) - - class OtEventListParser(EventListParser): def serialize( self, events: Iterable[Event], sections: Iterable[Section], file: Path diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index bd16b108a..242f67ded 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -23,7 +23,6 @@ Datastore, EventListParser, TrackParser, - TrackToVideoRepository, VideoParser, ) from OTAnalytics.application.eventlist import SceneActionDetector @@ -161,9 +160,6 @@ ExportTrackStatistics, TrackStatisticsExporterFactory, ) -from OTAnalytics.application.use_cases.track_to_video_repository import ( - ClearAllTrackToVideos, -) from OTAnalytics.application.use_cases.update_project import ProjectUpdater from OTAnalytics.application.use_cases.video_repository import ( AddAllVideos, @@ -238,7 +234,6 @@ OtFlowParser, OttrkFormatFixer, OttrkParser, - OttrkVideoParser, SimpleVideoParser, ) from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser @@ -438,10 +433,6 @@ def enable_filter_track_by_date(self) -> EnableFilterTrackByDate: self.track_view_state, self.filter_element_settings_restorer ) - @cached_property - def clear_all_track_to_videos(self) -> ClearAllTrackToVideos: - return ClearAllTrackToVideos(self.track_to_video_repository) - @cached_property def clear_all_videos(self) -> ClearAllVideos: return ClearAllVideos(self.video_repository) @@ -573,7 +564,6 @@ def datastore(self) -> Datastore: """ track_parser = self._create_track_parser() event_list_parser = self._create_event_list_parser() - track_video_parser = OttrkVideoParser(self.video_parser) return Datastore( self.track_repository, self.track_file_repository, @@ -582,10 +572,8 @@ def datastore(self) -> Datastore: self.flow_repository, self.event_repository, event_list_parser, - self.track_to_video_repository, self.video_repository, self.video_parser, - track_video_parser, self.progressbar_builder, self.remark_repository, ) @@ -798,7 +786,6 @@ def clear_all_repositories(self) -> ClearRepositories: self.clear_all_flows, self.clear_all_intersections, self.clear_all_sections, - self.clear_all_track_to_videos, self.clear_all_tracks, self.clear_all_videos, ) @@ -837,14 +824,12 @@ def cut_tracks_intersecting_section(self) -> CutTracksIntersectingSection: @cached_property def load_track_files(self) -> LoadTrackFiles: track_parser = self._create_track_parser() - track_video_parser = OttrkVideoParser(self.video_parser) return LoadTrackFiles( track_parser, - track_video_parser, self.track_repository, self.track_file_repository, self.video_repository, - self.track_to_video_repository, + self.video_parser, self.progressbar_builder, self.tracks_metadata, self.videos_metadata, @@ -862,10 +847,6 @@ def remark_repository(self) -> RemarkRepository: def video_repository(self) -> VideoRepository: return VideoRepository() - @cached_property - def track_to_video_repository(self) -> TrackToVideoRepository: - return TrackToVideoRepository() - @cached_property def preload_input_files(self) -> PreloadInputFiles: return PreloadInputFiles( diff --git a/tests/OTAnalytics/application/test_datastore.py b/tests/OTAnalytics/application/test_datastore.py index 5836d6341..b9826b737 100644 --- a/tests/OTAnalytics/application/test_datastore.py +++ b/tests/OTAnalytics/application/test_datastore.py @@ -11,8 +11,6 @@ Datastore, EventListParser, TrackParser, - TrackToVideoRepository, - TrackVideoParser, VideoParser, ) from OTAnalytics.application.parser.config_parser import ConfigParser @@ -115,11 +113,6 @@ def video_parser() -> Mock: return Mock(spec=VideoParser) -@pytest.fixture -def track_video_parser() -> Mock: - return Mock(spec=TrackVideoParser) - - @pytest.fixture def event_repository() -> Mock: return Mock(spec=EventRepository) @@ -135,11 +128,6 @@ def video_repository() -> Mock: return Mock(spec=VideoRepository) -@pytest.fixture -def track_to_video_repository() -> Mock: - return Mock(spec=TrackToVideoRepository) - - @pytest.fixture def config_parser() -> Mock: return Mock(spec=ConfigParser) @@ -185,8 +173,6 @@ def test_save_event_list_file( event_list_parser=event_list_parser, video_repository=video_repository, video_parser=video_parser, - track_video_parser=track_video_parser, - track_to_video_repository=track_to_video_repository, progressbar=progressbar, remark_repository=remark_repository, ) @@ -204,11 +190,9 @@ def test_update_section_plugin_data_not_existing( section_repository: Mock, flow_repository: Mock, video_parser: Mock, - track_video_parser: Mock, event_repository: Mock, event_list_parser: Mock, video_repository: Mock, - track_to_video_repository: Mock, progressbar: Mock, remark_repository: Mock, ) -> None: @@ -222,8 +206,6 @@ def test_update_section_plugin_data_not_existing( event_list_parser=event_list_parser, video_repository=video_repository, video_parser=video_parser, - track_video_parser=track_video_parser, - track_to_video_repository=track_to_video_repository, progressbar=progressbar, remark_repository=remark_repository, ) @@ -247,11 +229,9 @@ def test_update_section_plugin_data_with_existing_data( section_repository: Mock, flow_repository: Mock, video_parser: Mock, - track_video_parser: Mock, event_repository: Mock, event_list_parser: Mock, video_repository: Mock, - track_to_video_repository: Mock, progressbar: Mock, remark_repository: Mock, ) -> None: @@ -265,8 +245,6 @@ def test_update_section_plugin_data_with_existing_data( event_list_parser=event_list_parser, video_repository=video_repository, video_parser=video_parser, - track_video_parser=track_video_parser, - track_to_video_repository=track_to_video_repository, progressbar=progressbar, remark_repository=remark_repository, ) diff --git a/tests/OTAnalytics/application/use_cases/test_clear_repositories.py b/tests/OTAnalytics/application/use_cases/test_clear_repositories.py index 8a4b09842..57ec0c5dd 100644 --- a/tests/OTAnalytics/application/use_cases/test_clear_repositories.py +++ b/tests/OTAnalytics/application/use_cases/test_clear_repositories.py @@ -8,9 +8,6 @@ ) from OTAnalytics.application.use_cases.section_repository import ClearAllSections from OTAnalytics.application.use_cases.track_repository import ClearAllTracks -from OTAnalytics.application.use_cases.track_to_video_repository import ( - ClearAllTrackToVideos, -) from OTAnalytics.application.use_cases.video_repository import ClearAllVideos @@ -20,7 +17,6 @@ def test_clear_repositories(self) -> None: clear_all_flows = Mock(spec=ClearAllFlows) clear_all_intersections = Mock(spec=ClearAllIntersections) clear_all_sections = Mock(spec=ClearAllSections) - clear_all_track_to_videos = Mock(spec=ClearAllTrackToVideos) clear_all_tracks = Mock(spec=ClearAllTracks) clear_all_videos = Mock(spec=ClearAllVideos) @@ -29,7 +25,6 @@ def test_clear_repositories(self) -> None: clear_all_flows, clear_all_intersections, clear_all_sections, - clear_all_track_to_videos, clear_all_tracks, clear_all_videos, ) @@ -38,6 +33,5 @@ def test_clear_repositories(self) -> None: clear_all_flows.assert_called_once() clear_all_intersections.clear.assert_called_once() clear_all_sections.assert_called_once() - clear_all_track_to_videos.assert_called_once() clear_all_tracks.assert_called_once() clear_all_videos.assert_called_once() diff --git a/tests/OTAnalytics/application/use_cases/test_load_track_files.py b/tests/OTAnalytics/application/use_cases/test_load_track_files.py index 3e4725d60..9ea8af889 100644 --- a/tests/OTAnalytics/application/use_cases/test_load_track_files.py +++ b/tests/OTAnalytics/application/use_cases/test_load_track_files.py @@ -98,9 +98,8 @@ class Given: track_repository: Mock = Mock() track_file_repository: Mock = Mock() track_parser: Mock = Mock() - track_video_parser: Mock = Mock() video_repository: Mock = Mock() - track_to_video_repository: Mock = Mock() + video_parser: Mock = Mock() progressbar: Mock = Mock() tracks_metadata: Mock = Mock() videos_metadata: Mock = Mock() @@ -109,10 +108,9 @@ class Given: def __post_init__(self) -> None: self.order.track_parser = self.track_parser self.order.videos_metadata = self.videos_metadata - self.order.track_video_parser = self.track_video_parser self.order.video_repository = self.video_repository self.order.track_repository = self.track_repository - self.order.track_to_video_repository = self.track_to_video_repository + self.order.video_parser = self.video_parser self.order.tracks_metadata = self.tracks_metadata @@ -134,7 +132,6 @@ def setup( given = Given(track_ids, videos, classes, parse_result) given.track_file_repository.get_all.return_value = existing_track_files given.track_parser.parse.return_value = parse_result - given.track_video_parser.parse.return_value = track_ids, videos given.progressbar.return_value = track_files return given @@ -158,11 +155,10 @@ def create_detection_metadata(classes: set[str]) -> Mock: def create_target(given: Given) -> LoadTrackFiles: return LoadTrackFiles( track_parser=given.track_parser, - track_video_parser=given.track_video_parser, track_repository=given.track_repository, track_file_repository=given.track_file_repository, video_repository=given.video_repository, - track_to_video_repository=given.track_to_video_repository, + video_parser=given.video_parser, progressbar=given.progressbar, tracks_metadata=given.tracks_metadata, videos_metadata=given.videos_metadata, diff --git a/tests/OTAnalytics/application/use_cases/track_to_video_repository.py b/tests/OTAnalytics/application/use_cases/track_to_video_repository.py deleted file mode 100644 index e6e185b59..000000000 --- a/tests/OTAnalytics/application/use_cases/track_to_video_repository.py +++ /dev/null @@ -1,14 +0,0 @@ -from unittest.mock import Mock - -from OTAnalytics.application.datastore import TrackToVideoRepository -from OTAnalytics.application.use_cases.track_to_video_repository import ( - ClearAllTrackToVideos, -) - - -class TestClearAllTrackToVideos: - def test_clear(self) -> None: - repository = Mock(spec=TrackToVideoRepository) - clear_all_track_to_videos = ClearAllTrackToVideos(repository) - clear_all_track_to_videos() - repository.clear.assert_called_once() From 2a5191f045bda986bdecf92f410f0f3eec9c2a7b Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 21:40:58 +0200 Subject: [PATCH 035/128] Remove unnecessary code --- .../application/use_cases/load_track_files.py | 40 ----- .../use_cases/test_load_track_files.py | 164 +++++++++++++----- 2 files changed, 117 insertions(+), 87 deletions(-) diff --git a/OTAnalytics/application/use_cases/load_track_files.py b/OTAnalytics/application/use_cases/load_track_files.py index 4a62efc6d..0fa4b81c5 100644 --- a/OTAnalytics/application/use_cases/load_track_files.py +++ b/OTAnalytics/application/use_cases/load_track_files.py @@ -36,9 +36,6 @@ def __call__(self, files: list[Path]) -> None: Args: files (Path): files in ottrk format. """ - self._load_all_at_once(files) - - def _load_all_at_once(self, files: list[Path]) -> None: if not files: return parent_folder = files[0].parent @@ -68,46 +65,9 @@ def _load_all_at_once(self, files: list[Path]) -> None: ) logger().info(f"Loaded {len(files_to_load)} track files and videos...") - def _load_one_by_one(self, files: list[Path]) -> None: - raised_exceptions: list[Exception] = [] - for file in self._progressbar( - files, unit="files", description="Processed ottrk files: " - ): - if self._is_file_already_loaded(file): - logger().warning(f"File '{file}' already loaded. Skipping... ") - continue - try: - - self.load(file) - except Exception as cause: - raised_exceptions.append(cause) - if raised_exceptions: - raise ExceptionGroup( - "Errors occurred while loading the track files:", raised_exceptions - ) - def _is_file_already_loaded(self, file: Path) -> bool: return file in self._track_file_repository.get_all() - def load(self, file: Path) -> None: - """ - Load and parse the given track file together with the corresponding video file. - - Args: - file (Path): file in ottrk format - """ - parse_result = self._track_parser.parse(file) - video = self._video_parser.parse( - file.parent / parse_result.video_metadata.path, parse_result.video_metadata - ) - self._videos_metadata.update(parse_result.video_metadata) - self._video_repository.add(video) - self._track_repository.add_all(parse_result.tracks) - self._track_file_repository.add(file) - self._tracks_metadata.update_detection_classes( - parse_result.detection_metadata.detection_classes - ) - def _log_already_loaded_files( self, files: list[Path], files_to_load: list[Path] ) -> None: diff --git a/tests/OTAnalytics/application/use_cases/test_load_track_files.py b/tests/OTAnalytics/application/use_cases/test_load_track_files.py index 9ea8af889..76fc70628 100644 --- a/tests/OTAnalytics/application/use_cases/test_load_track_files.py +++ b/tests/OTAnalytics/application/use_cases/test_load_track_files.py @@ -1,8 +1,8 @@ from dataclasses import dataclass from pathlib import Path -from unittest.mock import MagicMock, Mock, call, patch +from unittest.mock import MagicMock, Mock -from OTAnalytics.application.datastore import TrackParseResult +from OTAnalytics.application.datastore import TracksParseResult from OTAnalytics.application.use_cases.load_track_files import LoadTrackFiles from OTAnalytics.domain.track import TrackId from OTAnalytics.domain.video import Video @@ -12,22 +12,28 @@ class TestLoadTrackFile: - @patch("OTAnalytics.application.use_cases.load_track_files.LoadTrackFiles.load") - def test_load_multiple_files(self, mock_load: Mock) -> None: + def test_load_multiple_files(self) -> None: given = setup( - track_ids=[], - video_files=[], + track_ids=[TrackId("1"), TrackId("2")], + video_files=[Path("video1.mp4"), Path("video2.mp4")], track_files=[some_file, other_file], existing_track_files=[], - classes=set(), + classes={"class1", "class2"}, ) target = create_target(given) target([some_file, other_file]) - assert mock_load.call_args_list == [call(some_file), call(other_file)] - @patch("OTAnalytics.application.use_cases.load_track_files.LoadTrackFiles.load") - def test_load_existing_track_file(self, mock_load: Mock) -> None: + given.track_parser.parse_files.assert_called_once_with([some_file, other_file]) + given.video_repository.add_all.assert_called_once_with(given.videos) + given.track_repository.add_all.assert_called_once_with( + given.parse_result.tracks + ) + given.track_file_repository.add_all.assert_called_once_with( + [some_file, other_file] + ) + + def test_load_existing_track_file(self) -> None: """ # Requirement https://openproject.platomo.de/projects/001-opentrafficcam-live/work_packages/2665 @@ -43,28 +49,73 @@ def test_load_existing_track_file(self, mock_load: Mock) -> None: target = create_target(given) target([some_file]) - mock_load.assert_not_called() - @patch("OTAnalytics.application.use_cases.load_track_files.LoadTrackFiles.load") - def test_load_multiple_with_existing_track_file(self, mock_load: Mock) -> None: + # Should not parse files if they are already loaded + given.track_parser.parse_files.assert_not_called() + given.video_repository.add_all.assert_not_called() + given.track_repository.add_all.assert_not_called() + given.track_file_repository.add_all.assert_not_called() + + def test_load_multiple_with_existing_track_file(self) -> None: """ # Requirement https://openproject.platomo.de/projects/001-opentrafficcam-live/work_packages/2665 @bug by randy-seng """ # noqa given = setup( - track_ids=[], - video_files=[], + track_ids=[TrackId("1")], + video_files=[Path("video1.mp4")], track_files=[some_file, other_file], existing_track_files=[some_file], - classes=set(), + classes={"class1"}, ) target = create_target(given) target([some_file, other_file]) - mock_load.assert_called_with(other_file) - def test_load(self) -> None: + # Should only parse the file that's not already loaded + given.track_parser.parse_files.assert_called_once_with([other_file]) + given.video_repository.add_all.assert_called_once_with(given.videos) + given.track_repository.add_all.assert_called_once_with( + given.parse_result.tracks + ) + given.track_file_repository.add_all.assert_called_once_with([other_file]) + + def test_load_empty_files_list(self) -> None: + given = setup( + track_ids=[], + video_files=[], + track_files=[], + existing_track_files=[], + classes=set(), + ) + target = create_target(given) + + target([]) + + # Should not call any methods when files list is empty + given.track_parser.parse_files.assert_not_called() + given.video_repository.add_all.assert_not_called() + given.track_repository.add_all.assert_not_called() + given.track_file_repository.add_all.assert_not_called() + + def test_load_with_videos_metadata_update(self) -> None: + given = setup( + track_ids=[TrackId("1")], + video_files=[Path("video1.mp4")], + track_files=[some_file], + existing_track_files=[], + classes={"class1"}, + ) + target = create_target(given) + + target([some_file]) + + # Should update videos metadata for each video + for video_metadata in given.parse_result.videos_metadata: + given.videos_metadata.update.assert_any_call(video_metadata) + + def test_load_with_detection_classes_update(self) -> None: given = setup( track_ids=[TrackId("1")], video_files=[Path("video1.mp4")], @@ -73,20 +124,14 @@ def test_load(self) -> None: classes={"class1", "class2"}, ) target = create_target(given) - target.load(some_file) - assert given.order.mock_calls == [ - call.track_parser.parse(some_file), - call.track_video_parser.parse( - some_file, given.track_ids, given.parse_result.video_metadata - ), - call.videos_metadata.update(given.parse_result.video_metadata), - call.video_repository.add_all(given.videos), - call.track_to_video_repository.add_all(given.track_ids, given.videos), - call.track_repository.add_all(given.parse_result.tracks), - call.tracks_metadata.update_detection_classes( - given.parse_result.detection_metadata.detection_classes - ), - ] + + target([some_file]) + + # Should update detection classes for each detection metadata + for detection_metadata in given.parse_result.detections_metadata: + given.tracks_metadata.update_detection_classes.assert_any_call( + detection_metadata.detection_classes + ) @dataclass @@ -94,16 +139,16 @@ class Given: track_ids: list[TrackId] videos: list[Video] classes: set[str] - parse_result: TrackParseResult - track_repository: Mock = Mock() - track_file_repository: Mock = Mock() - track_parser: Mock = Mock() - video_repository: Mock = Mock() - video_parser: Mock = Mock() - progressbar: Mock = Mock() - tracks_metadata: Mock = Mock() - videos_metadata: Mock = Mock() - order: MagicMock = MagicMock() + parse_result: TracksParseResult + track_repository: Mock + track_file_repository: Mock + track_parser: Mock + video_repository: Mock + video_parser: Mock + progressbar: Mock + tracks_metadata: Mock + videos_metadata: Mock + order: MagicMock def __post_init__(self) -> None: self.order.track_parser = self.track_parser @@ -122,16 +167,35 @@ def setup( classes: set[str], ) -> Given: videos = create_videos(video_files) - detection_metadata = create_detection_metadata(classes) + videos_metadata = [create_video_metadata(video_file) for video_file in video_files] + detections_metadata = [create_detection_metadata(classes) for _ in track_files] + track_dataset_result = Mock() type(track_dataset_result).track_ids = frozenset(track_ids) + parse_result = Mock() parse_result.tracks = track_dataset_result - parse_result.detection_metadata = detection_metadata - parse_result.video_metadata = Mock() - given = Given(track_ids, videos, classes, parse_result) + parse_result.videos_metadata = videos_metadata + parse_result.detections_metadata = detections_metadata + + given = Given( + track_ids=track_ids, + videos=videos, + classes=classes, + parse_result=parse_result, + track_repository=Mock(), + track_file_repository=Mock(), + track_parser=Mock(), + video_repository=Mock(), + video_parser=Mock(), + progressbar=Mock(), + tracks_metadata=Mock(), + videos_metadata=Mock(), + order=MagicMock(), + ) given.track_file_repository.get_all.return_value = existing_track_files - given.track_parser.parse.return_value = parse_result + given.track_parser.parse_files.return_value = parse_result + given.video_parser.parse.side_effect = videos given.progressbar.return_value = track_files return given @@ -146,6 +210,12 @@ def create_video(video_file: Path) -> Video: return video +def create_video_metadata(video_file: Path) -> Mock: + video_metadata = Mock() + video_metadata.path = video_file + return video_metadata + + def create_detection_metadata(classes: set[str]) -> Mock: detection_metadata = Mock() detection_metadata.detection_classes = classes From 111e4e17df72709731dd1bd4ba84bee9a60dd3b9 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 15 Aug 2025 21:41:06 +0200 Subject: [PATCH 036/128] Remove unnecessary code --- tests/OTAnalytics/application/test_datastore.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/OTAnalytics/application/test_datastore.py b/tests/OTAnalytics/application/test_datastore.py index b9826b737..ccae696de 100644 --- a/tests/OTAnalytics/application/test_datastore.py +++ b/tests/OTAnalytics/application/test_datastore.py @@ -153,16 +153,13 @@ def test_save_event_list_file( section_repository: Mock, flow_repository: Mock, video_parser: Mock, - track_video_parser: Mock, event_repository: Mock, event_list_parser: Mock, video_repository: Mock, - track_to_video_repository: Mock, progressbar: Mock, remark_repository: Mock, ) -> None: track_parser.parse.return_value = [] - track_video_parser.parse.return_value = [] store = Datastore( track_repository=track_repository, track_file_repository=track_file_repository, From db673f0d419659d9e373730ef2e0915ca55ebf2c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Sat, 16 Aug 2025 21:56:17 +0200 Subject: [PATCH 037/128] Extract convertion code into plugin --- .../convert_ottrk_to_feathers.py | 107 ++++++++++++++ convert_ottrk_to_feather.py | 134 +----------------- 2 files changed, 109 insertions(+), 132 deletions(-) create mode 100644 OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py diff --git a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py new file mode 100644 index 000000000..c956f71ff --- /dev/null +++ b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py @@ -0,0 +1,107 @@ +from pathlib import Path +from typing import Any, Dict + +from OTAnalytics.application.datastore import TrackParseResult +from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + PandasTrackGeometryDataset, +) +from OTAnalytics.plugin_datastore.track_store import ( + PandasByMaxConfidence, + PandasTrackDataset, +) +from OTAnalytics.plugin_parser.json_parser import write_json +from OTAnalytics.plugin_parser.otvision_parser import ( + DEFAULT_TRACK_LENGTH_LIMIT, + OttrkParser, +) +from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser + + +def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: + """Create a track geometry factory for PandasTrackDataset.""" + return PandasTrackGeometryDataset.from_track_dataset + + +def create_ottrk_parser() -> OttrkParser: + """Create an OttrkParser with the required detection parser.""" + track_classification_calculator = PandasByMaxConfidence() + track_geometry_factory = create_track_geometry_factory() + + detection_parser = PandasDetectionParser( + calculator=track_classification_calculator, + track_geometry_factory=track_geometry_factory, + track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, + ) + return OttrkParser(detection_parser) + + +def convert_to_pandas_dataset( + python_dataset: PythonTrackDataset, track_geometry_factory: TRACK_GEOMETRY_FACTORY +) -> PandasTrackDataset: + """Convert PythonTrackDataset to PandasTrackDataset.""" + tracks_list = python_dataset.as_list() + return PandasTrackDataset.from_list(tracks_list, track_geometry_factory) + + +def create_metadata_dict(parse_result: TrackParseResult) -> Dict[str, Any]: + """Create a metadata dictionary from TrackParseResult.""" + metadata = { + "detection_metadata": { + "detection_classes": list(parse_result.detection_metadata.detection_classes) + }, + "video_metadata": parse_result.video_metadata.to_dict(), + } + return metadata + + +def convert_ottrk_to_feather(input_file: Path) -> None: + """ + Convert an ottrk file to Apache Arrow/Feather format with metadata export. + + Args: + input_file: Path to the input ottrk file + """ + if not input_file.exists(): + raise FileNotFoundError(f"Input file not found: {input_file}") + + if not input_file.suffix.lower() == ".ottrk": + raise ValueError(f"Input file must have .ottrk extension: {input_file}") + + # Create output file paths using the same stem as input + output_stem = input_file.stem + output_dir = input_file.parent + feather_file = output_dir / f"{output_stem}.feather" + metadata_file = output_dir / f"{output_stem}_metadata.json" + + print(f"Reading ottrk file: {input_file}") + + # Parse the ottrk file + parser = create_ottrk_parser() + parse_result = parser.parse(input_file) + + print(f"Parsed {len(parse_result.tracks)} tracks") + + if isinstance(parse_result.tracks, PandasTrackDataset): + pandas_dataset = parse_result.tracks + else: + raise TypeError(f"Unsupported track dataset type: {type(parse_result.tracks)}") + + # Get the pandas DataFrame + df = pandas_dataset.get_data() + print(f"DataFrame shape: {df.shape}") + + # Save DataFrame to feather format + print(f"Saving DataFrame to: {feather_file}") + df.to_feather(feather_file) + + # Create and save metadata + metadata = create_metadata_dict(parse_result) + print(f"Saving metadata to: {metadata_file}") + write_json(metadata, metadata_file) + + print("Conversion completed successfully!") + print("Output files:") + print(f" - Data: {feather_file}") + print(f" - Metadata: {metadata_file}") diff --git a/convert_ottrk_to_feather.py b/convert_ottrk_to_feather.py index 945baeb55..9a20e98a6 100644 --- a/convert_ottrk_to_feather.py +++ b/convert_ottrk_to_feather.py @@ -30,138 +30,8 @@ import argparse import sys from pathlib import Path -from typing import Any, Dict - -from OTAnalytics.application.datastore import TrackParseResult -from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - PandasTrackGeometryDataset, -) -from OTAnalytics.plugin_datastore.track_store import ( - PandasByMaxConfidence, - PandasTrackDataset, -) -from OTAnalytics.plugin_parser.json_parser import write_json -from OTAnalytics.plugin_parser.otvision_parser import ( - DEFAULT_TRACK_LENGTH_LIMIT, - OttrkParser, -) -from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser - - -def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: - """Create a track geometry factory for PandasTrackDataset.""" - return PandasTrackGeometryDataset.from_track_dataset - - -def create_ottrk_parser() -> OttrkParser: - """Create an OttrkParser with the required detection parser.""" - # Create required dependencies for PythonDetectionParser - track_classification_calculator = PandasByMaxConfidence() - track_geometry_factory = create_track_geometry_factory() - - # Create detection parser with all required dependencies - detection_parser = PandasDetectionParser( - calculator=track_classification_calculator, - track_geometry_factory=track_geometry_factory, - track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, - ) - return OttrkParser(detection_parser) - - -def convert_to_pandas_dataset( - python_dataset: PythonTrackDataset, track_geometry_factory: TRACK_GEOMETRY_FACTORY -) -> PandasTrackDataset: - """Convert PythonTrackDataset to PandasTrackDataset.""" - tracks_list = python_dataset.as_list() - return PandasTrackDataset.from_list(tracks_list, track_geometry_factory) - - -def create_metadata_dict(parse_result: TrackParseResult) -> Dict[str, Any]: - """Create a metadata dictionary from TrackParseResult.""" - metadata = { - "detection_metadata": { - "detection_classes": list(parse_result.detection_metadata.detection_classes) - }, - "video_metadata": { - "path": parse_result.video_metadata.path, - "recorded_start_date": parse_result.video_metadata.recorded_start_date.isoformat(), # noqa - "recorded_fps": parse_result.video_metadata.recorded_fps, - "number_of_frames": parse_result.video_metadata.number_of_frames, - }, - } - - # Add optional fields if they exist - if parse_result.video_metadata.expected_duration is not None: - metadata["video_metadata"][ - "expected_duration" - ] = parse_result.video_metadata.expected_duration.total_seconds() - - if parse_result.video_metadata.actual_fps is not None: - metadata["video_metadata"][ - "actual_fps" - ] = parse_result.video_metadata.actual_fps - - return metadata - - -def convert_ottrk_to_feather(input_file: Path) -> None: - """ - Convert an ottrk file to Apache Arrow/Feather format with metadata export. - - Args: - input_file: Path to the input ottrk file - """ - if not input_file.exists(): - raise FileNotFoundError(f"Input file not found: {input_file}") - - if not input_file.suffix.lower() == ".ottrk": - raise ValueError(f"Input file must have .ottrk extension: {input_file}") - - # Create output file paths using the same stem as input - output_stem = input_file.stem - output_dir = input_file.parent - feather_file = output_dir / f"{output_stem}.feather" - metadata_file = output_dir / f"{output_stem}_metadata.json" - - print(f"Reading ottrk file: {input_file}") - - # Parse the ottrk file - parser = create_ottrk_parser() - parse_result = parser.parse(input_file) - - print(f"Parsed {len(parse_result.tracks)} tracks") - - # Convert to pandas dataset if needed - if isinstance(parse_result.tracks, PythonTrackDataset): - print("Converting to pandas dataset...") - track_geometry_factory = create_track_geometry_factory() - pandas_dataset = convert_to_pandas_dataset( - parse_result.tracks, track_geometry_factory - ) - elif isinstance(parse_result.tracks, PandasTrackDataset): - pandas_dataset = parse_result.tracks - else: - raise TypeError(f"Unsupported track dataset type: {type(parse_result.tracks)}") - - # Get the pandas DataFrame - df = pandas_dataset.get_data() - print(f"DataFrame shape: {df.shape}") - - # Save DataFrame to feather format - print(f"Saving DataFrame to: {feather_file}") - df.to_feather(feather_file) - - # Create and save metadata - metadata = create_metadata_dict(parse_result) - print(f"Saving metadata to: {metadata_file}") - write_json(metadata, metadata_file) - - print("Conversion completed successfully!") - print("Output files:") - print(f" - Data: {feather_file}") - print(f" - Metadata: {metadata_file}") + +from OTAnalytics.plugin_parser.convert_ottrk_to_feathers import convert_ottrk_to_feather def main(): From 45b61882ea94e4f4588be3c4493cee6a11e9db35 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Sat, 16 Aug 2025 21:58:32 +0200 Subject: [PATCH 038/128] Automatically convert files from ottrk to feathers format --- OTAnalytics/plugin_parser/feathers_parser.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_parser/feathers_parser.py b/OTAnalytics/plugin_parser/feathers_parser.py index 75ca32e44..6b28e1442 100644 --- a/OTAnalytics/plugin_parser/feathers_parser.py +++ b/OTAnalytics/plugin_parser/feathers_parser.py @@ -27,6 +27,7 @@ PandasByMaxConfidence, PandasTrackDataset, ) +from OTAnalytics.plugin_parser.convert_ottrk_to_feathers import convert_ottrk_to_feather from OTAnalytics.plugin_parser.json_parser import parse_json @@ -37,9 +38,12 @@ def use_feathers_files(files: list[Path]) -> list[Path]: try: if not file.suffix.lower() == ".feather": if file.suffix.lower() == ".ottrk": + convert_ottrk_to_feather(file) result.append(file.with_suffix(".feather")) else: - raise ValueError(f"Input file must have .feather extension: {file}") + raise ValueError( + f"Input file must have .feather or .ottrk extension: {file}" + ) except Exception as cause: raised_exceptions.append(cause) if raised_exceptions: From 2bd0aaf5d089a8c82f466f84fc3a5b8cb2c45c06 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 15:43:38 +0200 Subject: [PATCH 039/128] Add polars datasets --- .pre-commit-config.yaml | 1 + OTAnalytics/domain/track.py | 8 + .../plugin_datastore/polars_track_store.py | 878 +++++++++++++++ .../polars_geometry_store.py | 1001 +++++++++++++++++ requirements.txt | 1 + .../test_polars_track_store.py | 401 +++++++ .../test_geometry_store_comparison.py | 497 ++++++++ 7 files changed, 2787 insertions(+) create mode 100644 OTAnalytics/plugin_datastore/polars_track_store.py create mode 100644 OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py create mode 100644 tests/OTAnalytics/plugin_datastore/test_polars_track_store.py create mode 100644 tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d910b407f..2bf717553 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,6 +59,7 @@ repos: - numpy==2.1.1 - pandas-stubs==2.2.3.241126 - pandas==2.2.3 + - polars==1.32.3 - pre-commit==4.1.0 - pyarrow==21.0.0 - pytest-asyncio==0.24.0 diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index 067c91989..c5314c779 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -31,6 +31,14 @@ def __str__(self) -> str: return self.id +def unpack(track_id: TrackId | str) -> str: + return track_id.id if isinstance(track_id, TrackId) else track_id + + +def pack(track_id: TrackId | str) -> TrackId: + return track_id if isinstance(track_id, TrackId) else TrackId(track_id) + + class TrackError(Exception): def __init__(self, track_id: TrackId, *args: object) -> None: super().__init__(*args) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py new file mode 100644 index 000000000..44ab394fa --- /dev/null +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -0,0 +1,878 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import datetime +from math import ceil +from typing import ( + Any, + Callable, + Generator, + Iterable, + Iterator, + Optional, + Sequence, + cast, +) + +import numpy +import polars as pl +import shapely +from more_itertools import batched +from shapely.creation import prepare +from shapely.geometry.base import BaseGeometry +from shapely.vectorized import contains + +from OTAnalytics.application.logger import logger +from OTAnalytics.domain import track +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.section import Section, SectionId +from OTAnalytics.domain.track import Detection, Track, TrackId, unpack +from OTAnalytics.domain.track_dataset.track_dataset import ( + END_FRAME, + END_OCCURRENCE, + END_VIDEO_NAME, + END_X, + END_Y, + START_FRAME, + START_OCCURRENCE, + START_VIDEO_NAME, + START_X, + START_Y, + IntersectionPoint, + TrackDataset, + TrackDoesNotExistError, + TrackSegmentDataset, +) +from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + ROW_ID, + PolarsTrackGeometryDataset, +) +from OTAnalytics.plugin_parser import ottrk_dataformat + +RANK = "rank" +MINIMUM_DETECTIONS = 2 + + +class PolarsDetection(Detection): + def __init__(self, track_id: str, row_data: dict, occurrence: datetime): + self._track_id = track_id + self._occurrence = occurrence + self._data = row_data + + @property + def classification(self) -> str: + return self.__get_attribute(track.CLASSIFICATION) + + def __get_attribute(self, column: str) -> Any: + return self._data[column] + + @property + def confidence(self) -> float: + return self.__get_attribute(track.CONFIDENCE) + + @property + def x(self) -> float: + return self.__get_attribute(track.X) + + @property + def y(self) -> float: + return self.__get_attribute(track.Y) + + @property + def w(self) -> float: + return self.__get_attribute(track.W) + + @property + def h(self) -> float: + return self.__get_attribute(track.H) + + @property + def frame(self) -> int: + frame_number = self.__get_attribute(track.FRAME) + if isinstance(frame_number, numpy.int64): + return frame_number.item() + return frame_number + + @property + def occurrence(self) -> datetime: + return self._occurrence + + @property + def interpolated_detection(self) -> bool: + return self.__get_attribute(track.INTERPOLATED_DETECTION) + + @property + def track_id(self) -> TrackId: + return TrackId(self._track_id) + + @property + def video_name(self) -> str: + return self.__get_attribute(track.VIDEO_NAME) + + @property + def input_file(self) -> str: + return self.__get_attribute(track.INPUT_FILE) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, PolarsDetection): + return False + return ( + self._track_id == other._track_id + and self._occurrence == other._occurrence + and self._data == other._data + ) + + def __hash__(self) -> int: + return hash((self._track_id, self._occurrence)) + + def __str__(self) -> str: + return ( + f"PolarsDetection(track_id={self._track_id}, " + f"occurrence={self._occurrence}, " + f"x={self.x}, y={self.y})" + ) + + def __repr__(self) -> str: + return str(self) + + +@dataclass +class PolarsTrack(Track): + _id: str + _data: pl.DataFrame + + @property + def id(self) -> TrackId: + return TrackId(self._id) + + @property + def original_id(self) -> TrackId: + return TrackId(self._data.get_column(track.ORIGINAL_TRACK_ID)[0]) + + @property + def classification(self) -> str: + return self._data.get_column(track.TRACK_CLASSIFICATION)[0] + + @property + def detections(self) -> list[Detection]: + detections: list[Detection] = [] + for row in self._data.to_dicts(): + occurrence = row[track.OCCURRENCE] + detections.append(PolarsDetection(self._id, row, occurrence)) + return detections + + def get_detection(self, index: int) -> Detection: + row = self._data.row(index, named=True) + occurrence = row[track.OCCURRENCE] + return PolarsDetection(self._id, row, occurrence) + + @property + def first_detection(self) -> Detection: + row = self._data.row(0, named=True) + occurrence = row[track.OCCURRENCE] + return PolarsDetection(self._id, row, occurrence) + + @property + def last_detection(self) -> Detection: + row = self._data.row(-1, named=True) + occurrence = row[track.OCCURRENCE] + return PolarsDetection(self._id, row, occurrence) + + +class PolarsTrackClassificationCalculator(ABC): + @abstractmethod + def calculate(self, detections: pl.DataFrame) -> pl.DataFrame: + raise NotImplementedError + + +class PolarsByMaxConfidence(PolarsTrackClassificationCalculator): + def calculate(self, detections: pl.DataFrame) -> pl.DataFrame: + if detections.is_empty(): + return pl.DataFrame() + + # Group by track_id and classification, sum confidence, then get max per track + classifications = ( + detections.select([track.TRACK_ID, track.CLASSIFICATION, track.CONFIDENCE]) + .group_by([track.TRACK_ID, track.CLASSIFICATION]) + .agg(pl.col(track.CONFIDENCE).sum()) + .sort(track.CONFIDENCE) + .group_by(track.TRACK_ID) + .last() + .rename({track.CLASSIFICATION: track.TRACK_CLASSIFICATION}) + .select([track.TRACK_ID, track.TRACK_CLASSIFICATION]) + ) + return classifications + + +COLUMNS = [ + track.CLASSIFICATION, + track.CONFIDENCE, + track.X, + track.Y, + track.W, + track.H, + track.FRAME, + track.OCCURRENCE, + ottrk_dataformat.INPUT_FILE_PATH, + track.INTERPOLATED_DETECTION, + ottrk_dataformat.FIRST, + ottrk_dataformat.FINISHED, + track.VIDEO_NAME, + track.INPUT_FILE, + track.TRACK_ID, + track.TRACK_CLASSIFICATION, + track.ORIGINAL_TRACK_ID, +] +DEFAULT_CLASSIFICATOR = PolarsByMaxConfidence() +INDEX_NAMES = [track.TRACK_ID, track.OCCURRENCE] +LEVEL_TRACK_ID = track.TRACK_ID +LEVEL_OCCURRENCE = track.OCCURRENCE +CUT_INDICES = "CUT_INDICES" +TRACK_LENGTH = "TRACK_LENGTH" + + +class PolarsTrackSegmentDataset(TrackSegmentDataset): + def __init__(self, segments: pl.DataFrame) -> None: + self._segments = segments + + def __eq__(self, other: Any) -> bool: + if isinstance(other, PolarsTrackSegmentDataset): + return self._segments.equals(other._segments) + return False + + def __hash__(self) -> int: + return hash(str(self._segments)) + + def __repr__(self) -> str: + return repr(self._segments) + + def __str__(self) -> str: + return repr(self._segments) + + def apply(self, consumer: Callable[[dict], None]) -> None: + for row in self._segments.to_dicts(): + # Filter out None values to match pandas behavior + filtered_row = {k: v for k, v in row.items() if v is not None} + consumer(filtered_row) + + +class PolarsDataFrameProvider(ABC): + @abstractmethod + def get_data(self) -> pl.DataFrame: + pass + + +def drop_row_id(df: pl.DataFrame) -> pl.DataFrame: + return df.drop(ROW_ID, strict=False) + + +POLARS_TRACK_GEOMETRY_FACTORY = Callable[ + [TrackDataset, RelativeOffsetCoordinate], PolarsTrackGeometryDataset +] + + +class PolarsTrackDataset(TrackDataset, PolarsDataFrameProvider): + @property + def track_ids(self) -> frozenset[TrackId]: + if self._dataset.is_empty(): + return frozenset() + return frozenset( + TrackId(_id) for _id in self._dataset.get_column(track.TRACK_ID).unique() + ) + + @property + def first_occurrence(self) -> datetime | None: + if not len(self): + return None + return cast(datetime, self._dataset.get_column(track.OCCURRENCE).min()) + + @property + def last_occurrence(self) -> datetime | None: + if not len(self): + return None + return cast(datetime, self._dataset.get_column(track.OCCURRENCE).max()) + + @property + def classifications(self) -> frozenset[str]: + if not len(self): + return frozenset() + return frozenset(self._dataset.get_column(track.TRACK_CLASSIFICATION).unique()) + + @property + def empty(self) -> bool: + return self._dataset.is_empty() + + def __init__( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + dataset: pl.DataFrame | None = None, + geometry_datasets: ( + dict[RelativeOffsetCoordinate, PolarsTrackGeometryDataset] | None + ) = None, + calculator: PolarsTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, + ): + if dataset is not None: + self._dataset: pl.DataFrame = dataset + else: + self._dataset = create_empty_dataframe() + + self.calculator = calculator + self.track_geometry_factory = track_geometry_factory + if geometry_datasets is None: + self._geometry_datasets = dict[ + RelativeOffsetCoordinate, PolarsTrackGeometryDataset + ]() + else: + self._geometry_datasets = geometry_datasets + + def __iter__(self) -> Iterator[Track]: + yield from self.as_generator() + + def as_generator(self) -> Generator[Track, None, None]: + track_ids = self._dataset.get_column(LEVEL_TRACK_ID).unique() + for track_id in track_ids: + yield self._create_track_flyweight(track_id) + + @staticmethod + def from_list( + tracks: list[Track], + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + calculator: PolarsTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, + ) -> "PolarsTrackDataset": + return PolarsTrackDataset.from_dataframe( + _convert_tracks(tracks), + track_geometry_factory, + calculator=calculator, + ) + + @staticmethod + def from_dataframe( + tracks: pl.DataFrame, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + geometry_dataset: ( + dict[RelativeOffsetCoordinate, PolarsTrackGeometryDataset] | None + ) = None, + calculator: PolarsTrackClassificationCalculator = DEFAULT_CLASSIFICATOR, + ) -> "PolarsTrackDataset": + if tracks.is_empty(): + return PolarsTrackDataset(track_geometry_factory) + tracks = tracks.drop(ROW_ID, strict=False).with_row_index(ROW_ID) + result = _assign_track_classification(tracks, calculator) + return PolarsTrackDataset( + track_geometry_factory, + result, + geometry_datasets=geometry_dataset, + ) + + def add_all(self, other: Iterable[Track]) -> "PolarsTrackDataset": + new_tracks = self.__get_tracks(other) + if new_tracks.is_empty(): + return self + if self._dataset.is_empty(): + return PolarsTrackDataset.from_dataframe( + new_tracks, self.track_geometry_factory, calculator=self.calculator + ) + + # Ensure new_tracks has track classification before concatenating + new_tracks_with_classification = _assign_track_classification( + new_tracks, self.calculator + ) + + # Get all tracks (existing + new) and assign classification + combined_tracks = pl.concat( + [drop_row_id(self._dataset), drop_row_id(new_tracks_with_classification)] + ).sort(INDEX_NAMES) + + # Re-assign track classification to the combined dataset + updated_dataset = _assign_track_classification(combined_tracks, self.calculator) + + updated_geometry_dataset = self._add_to_geometry_dataset( + PolarsTrackDataset.from_dataframe( + updated_dataset, self.track_geometry_factory + ) + ) + return PolarsTrackDataset.from_dataframe( + updated_dataset, self.track_geometry_factory, updated_geometry_dataset + ) + + def __get_tracks(self, other: Iterable[Track]) -> pl.DataFrame: + if isinstance(other, PolarsDataFrameProvider): + return other.get_data() + else: + return _convert_tracks(other) + + def get_for(self, id: TrackId) -> Optional[Track]: + track_data = self._dataset.filter(pl.col(LEVEL_TRACK_ID) == unpack(id)) + if track_data.is_empty(): + return None + return self._create_track_flyweight(unpack(id)) + + def clear(self) -> "PolarsTrackDataset": + return PolarsTrackDataset(self.track_geometry_factory) + + def _create_track_flyweight(self, track_id: str) -> Track: + """Create a Track flyweight object for the given track_id.""" + track_data = self._dataset.filter(pl.col(LEVEL_TRACK_ID) == track_id) + return PolarsTrack(track_id, track_data) + + def remove(self, track_id: TrackId) -> "PolarsTrackDataset": + filtered_data = self._dataset.filter(pl.col(LEVEL_TRACK_ID) != unpack(track_id)) + return PolarsTrackDataset( + self.track_geometry_factory, + filtered_data, + self._geometry_datasets, + self.calculator, + ) + + def remove_multiple(self, track_ids: set[TrackId]) -> "PolarsTrackDataset": + track_id_strings = [unpack(track_id) for track_id in track_ids] + filtered_data = self._dataset.filter( + ~pl.col(LEVEL_TRACK_ID).is_in(track_id_strings) + ) + return PolarsTrackDataset( + self.track_geometry_factory, + filtered_data, + self._geometry_datasets, + self.calculator, + ) + + def as_list(self) -> list[Track]: + return list(self.as_generator()) + + def get_data(self) -> pl.DataFrame: + return self._dataset + + def __len__(self) -> int: + return self._dataset.get_column(LEVEL_TRACK_ID).n_unique() + + def intersecting_tracks( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> set[TrackId]: + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.intersecting_tracks(sections) + + def _get_geometry_dataset_for( + self, offset: RelativeOffsetCoordinate + ) -> PolarsTrackGeometryDataset: + """Retrieves track geometries for given offset. + + If offset does not exist, a new TrackGeometryDataset with the applied offset + will be created and saved. + + Args: + offset (RelativeOffsetCoordinate): the offset to retrieve track geometries + for. + + Returns: + TrackGeometryDataset: the track geometry dataset with the given offset + applied. + """ + if (geometry_dataset := self._geometry_datasets.get(offset, None)) is None: + geometry_dataset = self.track_geometry_factory(self, offset) + self._geometry_datasets[offset] = geometry_dataset + return geometry_dataset + + def intersection_points( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.intersection_points(sections) + + def ids_inside(self, sections: list[Section]) -> set[TrackId]: + result: set[TrackId] = set() + for _section in sections: + section_geom = area_section_to_shapely(_section) + offset = _section.get_offset(EventType.SECTION_ENTER) + x_coords = ( + self._dataset[track.X] + offset.x * self._dataset[track.W] + ).to_numpy() + y_coords = ( + self._dataset[track.Y] + offset.y * self._dataset[track.H] + ).to_numpy() + + contains_masks = contains(section_geom, x_coords, y_coords) + result.update( + self._dataset.filter(contains_masks) + .select(pl.col(track.TRACK_ID)) + .unique() + .to_series() + .to_list() + ) + return result + + def contained_by_sections( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: + raise NotImplementedError + + def _add_to_geometry_dataset(self, dataset: "PolarsTrackDataset") -> dict: + """Add to geometry dataset - placeholder implementation.""" + return self._geometry_datasets + + def calculate_geometries_for( + self, offsets: Iterable[RelativeOffsetCoordinate] + ) -> None: + for offset in offsets: + if offset not in self._geometry_datasets.keys(): + self._geometry_datasets[offset] = self._get_geometry_dataset_for(offset) + + def get_first_segments(self) -> TrackSegmentDataset: + segments = self.__create_segments() + if segments.is_empty(): + return PolarsTrackSegmentDataset(segments) + + first_segments = segments.group_by(LEVEL_TRACK_ID, maintain_order=True).first() + return PolarsTrackSegmentDataset(first_segments) + + def get_last_segments(self) -> TrackSegmentDataset: + segments = self.__create_segments() + if segments.is_empty(): + return PolarsTrackSegmentDataset(segments) + + last_segments = segments.group_by(LEVEL_TRACK_ID, maintain_order=True).last() + return PolarsTrackSegmentDataset(last_segments) + + def __create_segments(self) -> pl.DataFrame: + """Create track segments from detections.""" + if self._dataset.is_empty(): + schema = { + track.TRACK_ID: pl.Utf8, + track.OCCURRENCE: pl.Datetime, + track.TRACK_CLASSIFICATION: pl.Utf8, + START_X: pl.Float64, + START_Y: pl.Float64, + START_OCCURRENCE: pl.Datetime, + START_FRAME: pl.Int64, + START_VIDEO_NAME: pl.Utf8, + END_X: pl.Float64, + END_Y: pl.Float64, + END_OCCURRENCE: pl.Datetime, + END_FRAME: pl.Int64, + END_VIDEO_NAME: pl.Utf8, + } + return pl.DataFrame(schema=schema) + + data = self._dataset.sort([LEVEL_TRACK_ID, LEVEL_OCCURRENCE]) + + # Create shifted columns for start positions + segments = ( + data.with_columns( + [ + pl.col(track.X).shift(1).over(LEVEL_TRACK_ID).alias(START_X), + pl.col(track.Y).shift(1).over(LEVEL_TRACK_ID).alias(START_Y), + pl.col(track.OCCURRENCE) + .shift(1) + .over(LEVEL_TRACK_ID) + .alias(START_OCCURRENCE), + pl.col(track.FRAME) + .shift(1) + .over(LEVEL_TRACK_ID) + .alias(START_FRAME), + pl.col(track.VIDEO_NAME) + .shift(1) + .over(LEVEL_TRACK_ID) + .alias(START_VIDEO_NAME), + ] + ) + .drop_nulls( + subset=[ + START_X, + START_Y, + START_OCCURRENCE, + START_FRAME, + START_VIDEO_NAME, + ] + ) + .rename( + { + track.X: END_X, + track.Y: END_Y, + track.OCCURRENCE: END_OCCURRENCE, + track.FRAME: END_FRAME, + track.VIDEO_NAME: END_VIDEO_NAME, + } + ) + .select( + [ + track.TRACK_ID, + track.TRACK_CLASSIFICATION, + START_X, + START_Y, + START_OCCURRENCE, + START_FRAME, + START_VIDEO_NAME, + END_X, + END_Y, + END_OCCURRENCE, + END_FRAME, + END_VIDEO_NAME, + ] + ) + ) + return segments + + def cut_with_section( + self, section: Section, offset: RelativeOffsetCoordinate + ) -> tuple["TrackDataset", set[TrackId]]: + if len(self) == 0: + logger().info("No tracks to cut") + return self, set() + + geometry_dataset = self._get_geometry_dataset_for(offset) + new_track_ids = geometry_dataset.track_ids_after_cut(section) + old_track_ids = self._dataset.rename({track.TRACK_ID: "old_track_id"}) + + result = old_track_ids.join(new_track_ids, on=ROW_ID, how="left") + result = result.with_columns( + (pl.col(track.TRACK_ID).backward_fill().over("old_track_id")) + ).drop("old_track_id") + return ( + PolarsTrackDataset.from_dataframe(result, self.track_geometry_factory), + set(), + ) + + def _add_cut_track_part( + self, + cut_tracks_data: list, + detections: list[dict], + original_track_id_str: str, + original_id: str, + part_index: int, + ) -> None: + """Add a cut track part to the cut tracks data list.""" + new_track_id = f"{original_track_id_str}_{part_index}" + + for detection in detections: + # Create a copy of detection with new track ID + detection_copy = detection.copy() + detection_copy[track.TRACK_ID] = new_track_id + detection_copy[track.ORIGINAL_TRACK_ID] = original_id + cut_tracks_data.append(detection_copy) + + def filter_by_min_detection_length(self, length: int) -> TrackDataset: + if self._dataset.is_empty(): + return self + + detection_counts = self._dataset.group_by(LEVEL_TRACK_ID).agg( + pl.len().alias("count") + ) + + valid_track_ids = ( + detection_counts.filter(pl.col("count") >= length) + .get_column(LEVEL_TRACK_ID) + .to_list() + ) + + filtered_dataset = self._dataset.filter( + pl.col(LEVEL_TRACK_ID).is_in(valid_track_ids) + ) + + return PolarsTrackDataset( + self.track_geometry_factory, filtered_dataset, calculator=self.calculator + ) + + def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: + if not track_ids or self._dataset.is_empty(): + return {} + + try: + result = ( + self._dataset.filter(pl.col(LEVEL_TRACK_ID).is_in(track_ids)) + .group_by(LEVEL_TRACK_ID) + .agg(pl.col(track.CONFIDENCE).max()) + ) + + if result.is_empty(): + raise TrackDoesNotExistError( + "Some tracks do not exist in dataset with given id" + ) + + return dict( + zip( + result.get_column(LEVEL_TRACK_ID).to_list(), + result.get_column(track.CONFIDENCE).to_list(), + ) + ) + except Exception as cause: + raise TrackDoesNotExistError( + "Some tracks do not exist in dataset with given id" + ) from cause + + def revert_cuts_for( + self, original_track_ids: frozenset[TrackId] + ) -> "PolarsTrackDataset": + if self._dataset.is_empty(): + return self + + ids_to_revert = self._get_existing_track_ids(original_track_ids) + if not ids_to_revert: + return self + + # Revert track IDs back to original IDs + result = self._dataset.with_columns( + pl.when(pl.col(LEVEL_TRACK_ID).is_in(ids_to_revert)) + .then(pl.col(track.ORIGINAL_TRACK_ID)) + .otherwise(pl.col(LEVEL_TRACK_ID)) + .alias(LEVEL_TRACK_ID) + ) + + return PolarsTrackDataset.from_dataframe( + result, + self.track_geometry_factory, + geometry_dataset=self._geometry_datasets, + calculator=self.calculator, + ) + + def _get_existing_track_ids(self, track_ids: frozenset[TrackId]) -> list[str]: + if self._dataset.is_empty(): + return [] + + converted_ids = [track_id.id for track_id in track_ids] + existing_ids = ( + self._dataset.filter(pl.col(track.ORIGINAL_TRACK_ID).is_in(converted_ids)) + .get_column(LEVEL_TRACK_ID) + .unique() + .to_list() + ) + return existing_ids + + def split(self, batches: int) -> Sequence["PolarsTrackDataset"]: + dataset_size = len(self) + if dataset_size == 0: + return [self] + + batch_size = ceil(dataset_size / batches) + track_ids = self._dataset.get_column(LEVEL_TRACK_ID).unique().to_list() + + new_batches = [] + for batch_track_ids in batched(track_ids, batch_size): + batch_ids_list = list(batch_track_ids) + batch_dataset = self._dataset.filter( + pl.col(LEVEL_TRACK_ID).is_in(batch_ids_list) + ) + batch_geometries = self._get_geometries_for(batch_ids_list) + new_batches.append( + PolarsTrackDataset.from_dataframe( + batch_dataset, + self.track_geometry_factory, + batch_geometries, + calculator=self.calculator, + ) + ) + return new_batches + + def _get_geometries_for( + self, track_ids: list[str] + ) -> dict[RelativeOffsetCoordinate, PolarsTrackGeometryDataset]: + geometry_datasets: dict[ + RelativeOffsetCoordinate, PolarsTrackGeometryDataset + ] = {} + for offset, geometry_dataset in self._geometry_datasets.items(): + geometry_datasets[offset] = geometry_dataset.get_for(track_ids) + return geometry_datasets + + +def create_empty_dataframe() -> pl.DataFrame: + """Create an empty Polars DataFrame with the required schema.""" + specified_columns = [ + track.X, + track.Y, + track.W, + track.H, + track.FRAME, + track.CONFIDENCE, + track.INTERPOLATED_DETECTION, + track.OCCURRENCE, + ] + unspecified_columns = set(COLUMNS) - set(specified_columns) + schema: dict[str, type] = {} + for col in unspecified_columns: + schema[col] = pl.Utf8 + # Set specific data types for known columns + schema[track.X] = pl.Float64 + schema[track.Y] = pl.Float64 + schema[track.W] = pl.Float64 + schema[track.H] = pl.Float64 + schema[track.FRAME] = pl.Int64 + schema[track.CONFIDENCE] = pl.Float64 + schema[track.INTERPOLATED_DETECTION] = pl.Boolean + schema[track.OCCURRENCE] = pl.Datetime + + return pl.DataFrame(schema=schema) + + +def _convert_tracks(tracks: Iterable[Track]) -> pl.DataFrame: + """ + Convert tracks into a Polars dataframe. + + Args: + tracks (Iterable[Track]): tracks to convert. + + Returns: + pl.DataFrame: tracks as dataframe. + """ + if not tracks: + return create_empty_dataframe() + + prepared: list[dict] = [] + for current_track in list(tracks): + for detection in current_track.detections: + dto = detection.to_dict() + dto[track.ORIGINAL_TRACK_ID] = current_track.original_id.id + prepared.append(dto) + + if not prepared: + return pl.DataFrame(prepared) + + df = pl.DataFrame(prepared) + return _sort_tracks(df) + + +def _sort_tracks(track_df: pl.DataFrame) -> pl.DataFrame: + """Sort the given dataframe by trackId and occurrence. + + Args: + track_df (pl.DataFrame): dataframe of tracks + + Returns: + pl.DataFrame: sorted dataframe by track id and occurrence + """ + return track_df.sort(INDEX_NAMES) + + +def _assign_track_classification( + data: pl.DataFrame, calculator: PolarsTrackClassificationCalculator +) -> pl.DataFrame: + dropped = _drop_track_classification(data) + classification_per_track = calculator.calculate(dropped) + return dropped.join(classification_per_track, on=track.TRACK_ID, how="left") + + +def _drop_track_classification(data: pl.DataFrame) -> pl.DataFrame: + if track.TRACK_CLASSIFICATION in data.columns: + return data.drop(track.TRACK_CLASSIFICATION) + return data + + +def get_rows_by_track_ids(dataframe: pl.DataFrame, track_ids: list) -> pl.DataFrame: + """ + Get all rows of a DataFrame corresponding to the given track_ids. + + Args: + dataframe (pl.DataFrame): DataFrame with track_ids + track_ids (list[str]): List of track_ids to filter by + + Returns: + pl.DataFrame: Filtered DataFrame containing only rows with the given track_ids + """ + if dataframe.is_empty() or len(track_ids) == 0: + return create_empty_dataframe() + + # Filter the DataFrame to include only rows with track_ids in the provided list + return dataframe.filter(pl.col(LEVEL_TRACK_ID).is_in(track_ids)) + + +def area_section_to_shapely(section: Section) -> BaseGeometry: + geometry = shapely.Polygon([(c.x, c.y) for c in section.get_coordinates()]) + prepare(geometry) + return geometry diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py new file mode 100644 index 000000000..69de20570 --- /dev/null +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -0,0 +1,1001 @@ +import math +from dataclasses import dataclass +from typing import Any, Iterable, Optional, Sequence + +import polars as pl + +from OTAnalytics.domain import track +from OTAnalytics.domain.common import DataclassValidation +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.section import Section, SectionId, SectionType +from OTAnalytics.domain.track import OCCURRENCE, TRACK_ID, H, Track, TrackId, W, X, Y +from OTAnalytics.domain.track_dataset.track_dataset import ( + IntersectionPoint, + TrackDataset, + TrackGeometryDataset, +) +from OTAnalytics.domain.types import EventType + +# Column names for track segments +ROW_ID = "row_id" +START_OCCURRENCE = "start-occurrence" +END_OCCURRENCE = "end-occurrence" +START_X = "start-x" +START_Y = "start-y" +START_W = "start-w" +START_H = "start-h" +END_X = "end-x" +END_Y = "end-y" +END_W = "end-w" +END_H = "end-h" + +# Column names for intersections +INTERSECTS = "intersects" +INTERSECTION_X = "intersection_x" +INTERSECTION_Y = "intersection_y" +INTERSECTION_LINE_ID = "intersection_line_id" +INTERSECTS_POLYGON = "intersects-polygon" + +# Column names for intersection parameters +DENOMINATOR = "denominator" +NON_PARALLEL = "non_parallel" +UA = "ua" +UB = "ub" + + +@dataclass(frozen=True) +class Polygon(DataclassValidation): + """A polygon consists of an ordered sequence of coordinates. + + Note: + The coordinates are not required to form a closed chain in this representation. + + Args: + coordinates (list[Coordinate]): The coordinates defining the polygon. Must + contain at least two coordinates. + + Raises: + ValueError: If fewer than two coordinates are provided. + """ + + coordinates: list[Coordinate] + + def _validate(self) -> None: + if len(self.coordinates) < 2: + raise ValueError( + ( + "Number of coordinates to define a valid polygon must be " + f"greater equal two, but is {len(self.coordinates)}" + ) + ) + + +def create_track_segments(df: pl.DataFrame) -> pl.DataFrame: + """ + Create track segments from a DataFrame of track points. + + A track segment connects two consecutive points of a track. + + Args: + df (pl.DataFrame): DataFrame with columns TRACK_ID, OCCURRENCE, X, Y, W, H. + + Returns: + pl.DataFrame: DataFrame with columns TRACK_ID, START_OCCURRENCE, END_OCCURRENCE, + START_X, START_Y, START_W, START_H, END_X, END_Y, END_W, END_H. + """ + if df.is_empty(): + return pl.DataFrame() + + # Sort by track_id and occurrence for efficient window operations + df_sorted = df.sort([TRACK_ID, OCCURRENCE]) + + # Use window functions to create segments - this is much more efficient in polars + segments = df_sorted.with_columns( + [ + # End values (current row) + pl.col(OCCURRENCE).alias(END_OCCURRENCE), + pl.col(X).alias(END_X), + pl.col(Y).alias(END_Y), + pl.col(W).alias(END_W), + pl.col(H).alias(END_H), + # Start values (previous row within the same track) + pl.col(OCCURRENCE).shift(1).over(TRACK_ID).alias(START_OCCURRENCE), + pl.col(X).shift(1).over(TRACK_ID).alias(START_X), + pl.col(Y).shift(1).over(TRACK_ID).alias(START_Y), + pl.col(W).shift(1).over(TRACK_ID).alias(START_W), + pl.col(H).shift(1).over(TRACK_ID).alias(START_H), + ] + ).drop_nulls() # Remove rows where shift resulted in null (first rows per track) + + # Select only the required columns in the exact order that pandas produces + segments = segments.select( + [ + ROW_ID, + TRACK_ID, + END_OCCURRENCE, + END_X, + END_Y, + END_W, + END_H, + START_OCCURRENCE, + START_X, + START_Y, + START_W, + START_H, + ] + ) + + return segments + + +def calculate_intersection_parameters( + segments_df: pl.DataFrame, + line_x1: float, + line_y1: float, + line_x2: float, + line_y2: float, + offset: RelativeOffsetCoordinate, +) -> pl.DataFrame: + """ + Calculate intersection parameters between track segments and a line. + + This function implements the line-line intersection algorithm to determine + intersection parameters for segments and a reference line. + + Args: + segments_df (pl.DataFrame): DataFrame with track segments. + line_x1, line_y1, line_x2, line_y2 (float): Coordinates of the reference line. + offset (RelativeOffsetCoordinate): Offset applied to segment endpoints. + + Returns: + pl.DataFrame: DataFrame with intersection parameters. + """ + if segments_df.is_empty(): + return segments_df + + # Apply offset to segment endpoints + result_df = segments_df.with_columns( + [ + (pl.col(START_X) + pl.col(START_W) * offset.x).alias("seg_x1"), + (pl.col(START_Y) + pl.col(START_H) * offset.y).alias("seg_y1"), + (pl.col(END_X) + pl.col(END_W) * offset.x).alias("seg_x2"), + (pl.col(END_Y) + pl.col(END_H) * offset.y).alias("seg_y2"), + ] + ) + + # Calculate intersection parameters using vectorized operations + result_df = ( + result_df.with_columns( + [ + # Line direction vectors + pl.lit(line_x2 - line_x1).alias("line_dx"), + pl.lit(line_y2 - line_y1).alias("line_dy"), + (pl.col("seg_x2") - pl.col("seg_x1")).alias("seg_dx"), + (pl.col("seg_y2") - pl.col("seg_y1")).alias("seg_dy"), + ] + ) + .with_columns( + [ + # Calculate denominator for intersection formula + ( + pl.col("line_dx") * pl.col("seg_dy") + - pl.col("line_dy") * pl.col("seg_dx") + ).alias(DENOMINATOR), + ] + ) + .with_columns( + [ + # Check if lines are non-parallel + (pl.col(DENOMINATOR).abs() > 1e-10).alias(NON_PARALLEL), + ] + ) + ) + + # Calculate intersection parameters only for non-parallel segments + result_df = result_df.with_columns( + [ + pl.when(pl.col(NON_PARALLEL)) + .then( + ( + (pl.col("seg_x1") - line_x1) * pl.col("seg_dy") + - (pl.col("seg_y1") - line_y1) * pl.col("seg_dx") + ) + / pl.col(DENOMINATOR) + ) + .otherwise(None) + .alias(UA), + pl.when(pl.col(NON_PARALLEL)) + .then( + ( + (pl.col("seg_x1") - line_x1) * pl.col("line_dy") + - (pl.col("seg_y1") - line_y1) * pl.col("line_dx") + ) + / pl.col(DENOMINATOR) + ) + .otherwise(None) + .alias(UB), + ] + ) + + return result_df + + +def check_line_intersections( + segments_df: pl.DataFrame, + line_x1: float, + line_y1: float, + line_x2: float, + line_y2: float, + offset: RelativeOffsetCoordinate, +) -> pl.DataFrame: + """ + Check if track segments intersect with a line segment. + + Args: + segments_df (pl.DataFrame): DataFrame with track segments. + line_x1, line_y1, line_x2, line_y2 (float): Coordinates of the line segment. + offset (RelativeOffsetCoordinate): Offset applied to segment endpoints. + + Returns: + pl.DataFrame: DataFrame with intersection information. + """ + if segments_df.is_empty(): + return segments_df + + # Calculate intersection parameters + result_df = calculate_intersection_parameters( + segments_df, line_x1, line_y1, line_x2, line_y2, offset + ) + + # Check if intersection is within both line segments (0 <= ua <= 1 and 0 <= ub <= 1) + result_df = result_df.with_columns( + [ + ( + pl.col(NON_PARALLEL) + & (pl.col(UA).is_not_null()) + & (pl.col(UB).is_not_null()) + & (pl.col(UA) >= 0) + & (pl.col(UA) <= 1) + & (pl.col(UB) >= 0) + & (pl.col(UB) <= 1) + ).alias(INTERSECTS) + ] + ) + + return result_df + + +def calculate_intersection_points( + segments_df: pl.DataFrame, + line_x1: float, + line_y1: float, + line_x2: float, + line_y2: float, + offset: RelativeOffsetCoordinate, +) -> pl.DataFrame: + """ + Calculate intersection points between track segments and a line. + + Args: + segments_df (pl.DataFrame): DataFrame with track segments. + line_x1, line_y1, line_x2, line_y2 (float): Coordinates of the line. + offset (RelativeOffsetCoordinate): Offset applied to segment endpoints. + + Returns: + pl.DataFrame: DataFrame with intersection points. + """ + if segments_df.is_empty(): + return segments_df + + # First check for intersections + result_df = check_line_intersections( + segments_df, line_x1, line_y1, line_x2, line_y2, offset + ) + + # Calculate intersection coordinates for intersecting segments + result_df = result_df.with_columns( + [ + pl.when(pl.col(INTERSECTS)) + .then(line_x1 + pl.col(UA) * (line_x2 - line_x1)) + .otherwise(None) + .alias(INTERSECTION_X), + pl.when(pl.col(INTERSECTS)) + .then(line_y1 + pl.col(UA) * (line_y2 - line_y1)) + .otherwise(None) + .alias(INTERSECTION_Y), + ] + ) + + return result_df + + +def find_line_intersections( + segments_df: pl.DataFrame, + line_id: str, + start_x: float, + start_y: float, + end_x: float, + end_y: float, + offset: RelativeOffsetCoordinate | None = None, +) -> pl.DataFrame: + """ + Find intersections between track segments and a line segment. + + Args: + segments_df (pl.DataFrame): DataFrame with track segments. + line_id (str): Identifier for the line. + start_x, start_y, end_x, end_y (float): Line segment coordinates. + offset (RelativeOffsetCoordinate, optional): Offset for segment endpoints. + + Returns: + pl.DataFrame: DataFrame with intersection information and points. + """ + if segments_df.is_empty(): + return segments_df + + if offset is None: + offset = RelativeOffsetCoordinate(x=0.0, y=0.0) + + # Calculate intersection points + result_df = calculate_intersection_points( + segments_df, start_x, start_y, end_x, end_y, offset + ) + + # Add line ID for intersecting segments + result_df = result_df.with_columns( + [ + pl.when(pl.col(INTERSECTS)) + .then(pl.lit(line_id)) + .otherwise(None) + .alias(INTERSECTION_LINE_ID) + ] + ).select( + [ + track.TRACK_ID, + START_X, + START_Y, + END_X, + END_Y, + START_W, + START_H, + END_W, + END_H, + START_OCCURRENCE, + END_OCCURRENCE, + INTERSECTS, + INTERSECTION_X, + INTERSECTION_Y, + INTERSECTION_LINE_ID, + ] + ) + + return result_df + + +def check_polygon_intersections( + segments_df: pl.DataFrame, + polygon: Polygon, + offset: RelativeOffsetCoordinate, +) -> pl.DataFrame: + """ + Check if track segments intersect with a polygon. + + A polygon consists of a set of lines. This function checks if each track segment + intersects with any of the lines in the polygon. + + Args: + segments_df (pl.DataFrame): DataFrame with track segments. + polygon (Polygon): The polygon to check intersections with. + offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints. + + Returns: + pl.DataFrame: DataFrame with an additional column INTERSECTS_POLYGON. + """ + if segments_df.is_empty(): + return segments_df + + # Initialize intersection column + result_df = segments_df.with_columns([pl.lit(False).alias(INTERSECTS_POLYGON)]) + + # Get polygon coordinates + polygon_coordinates = polygon.coordinates + + # Calculate polygon bounding box + poly_min_x = min(c.x for c in polygon_coordinates) + poly_max_x = max(c.x for c in polygon_coordinates) + poly_min_y = min(c.y for c in polygon_coordinates) + poly_max_y = max(c.y for c in polygon_coordinates) + + # Calculate segment endpoints with offset + result_df = result_df.with_columns( + [ + (pl.col(START_X) + pl.col(START_W) * offset.x).alias("sxa"), + (pl.col(START_Y) + pl.col(START_H) * offset.y).alias("sya"), + (pl.col(END_X) + pl.col(END_W) * offset.x).alias("exa"), + (pl.col(END_Y) + pl.col(END_H) * offset.y).alias("eya"), + ] + ) + + # Calculate segment bounding boxes + result_df = result_df.with_columns( + [ + pl.min_horizontal([pl.col("sxa"), pl.col("exa")]).alias("seg_min_x"), + pl.max_horizontal([pl.col("sxa"), pl.col("exa")]).alias("seg_max_x"), + pl.min_horizontal([pl.col("sya"), pl.col("eya")]).alias("seg_min_y"), + pl.max_horizontal([pl.col("sya"), pl.col("eya")]).alias("seg_max_y"), + ] + ) + + # Pre-filter segments whose bounding box intersects polygon bounding box + candidate_mask = ( + (pl.col("seg_max_x") >= poly_min_x) + & (pl.col("seg_min_x") <= poly_max_x) + & (pl.col("seg_max_y") >= poly_min_y) + & (pl.col("seg_min_y") <= poly_max_y) + ) + + candidates_df = result_df.filter(candidate_mask) + + if candidates_df.is_empty(): + return result_df + + # Check intersections with each line segment of the polygon + intersecting_indices = set() + + for i in range(len(polygon_coordinates) - 1): + # Get the start and end points of the current line segment + start_x = polygon_coordinates[i].x + start_y = polygon_coordinates[i].y + end_x = polygon_coordinates[i + 1].x + end_y = polygon_coordinates[i + 1].y + + # Further prefilter by line bounding box + line_min_x = min(start_x, end_x) + line_max_x = max(start_x, end_x) + line_min_y = min(start_y, end_y) + line_max_y = max(start_y, end_y) + + line_candidates = candidates_df.filter( + (pl.col("seg_max_x") >= line_min_x) + & (pl.col("seg_min_x") <= line_max_x) + & (pl.col("seg_max_y") >= line_min_y) + & (pl.col("seg_min_y") <= line_max_y) + ) + + if line_candidates.is_empty(): + continue + + # Check which segments intersect with the current line segment + intersects_df = check_line_intersections( + line_candidates, start_x, start_y, end_x, end_y, offset + ) + + # Collect indices of intersecting segments + intersecting_rows = intersects_df.filter(pl.col(INTERSECTS)) + if not intersecting_rows.is_empty(): + # Get original indices from the filtered dataframe + for row in intersecting_rows.iter_rows(named=True): + # Find matching rows in original dataframe to get correct indices + matching_indices = ( + result_df.with_row_index() + .filter( + (pl.col(TRACK_ID) == row[TRACK_ID]) + & (pl.col(START_OCCURRENCE) == row[START_OCCURRENCE]) + & (pl.col(END_OCCURRENCE) == row[END_OCCURRENCE]) + ) + .select("index") + .to_series() + .to_list() + ) + intersecting_indices.update(matching_indices) + + # Update intersection column for identified segments + if intersecting_indices: + # Create a boolean mask for intersecting rows + mask_values = [i in intersecting_indices for i in range(len(result_df))] + result_df = result_df.with_columns( + [ + pl.when(pl.Series(mask_values)) + .then(True) + .otherwise(pl.col(INTERSECTS_POLYGON)) + .alias(INTERSECTS_POLYGON) + ] + ) + + return result_df.drop( + ["sxa", "sya", "exa", "eya", "seg_min_x", "seg_max_x", "seg_min_y", "seg_max_y"] + ) + + +def get_section_offset(section: Section) -> RelativeOffsetCoordinate: + if offset := section.relative_offset_coordinates.get(EventType.SECTION_ENTER): + return offset + raise ValueError(f"Section {section.id} has no offset") + + +class PolarsTrackGeometryDataset(TrackGeometryDataset): + _segments_df: pl.DataFrame + + def __init__( + self, + offset: RelativeOffsetCoordinate, + segments_df: Optional[pl.DataFrame] = None, + ): + """Initialize a PolarsTrackGeometryDataset. + + Args: + offset (RelativeOffsetCoordinate): Relative offset to apply to track points. + segments_df (Optional[pl.DataFrame], optional): DataFrame with track + segments. If None, an empty DataFrame will be created. Defaults to None. + """ + self._offset = offset + if segments_df is None: + self._segments_df = pl.DataFrame() + else: + self._segments_df = segments_df + + # Ensure the DataFrame has the required columns + required_columns = [ + TRACK_ID, + START_X, + START_Y, + START_W, + START_H, + END_X, + END_Y, + END_W, + END_H, + START_OCCURRENCE, + END_OCCURRENCE, + ] + + if not self._segments_df.is_empty(): + missing_columns = [ + col for col in required_columns if col not in self._segments_df.columns + ] + if missing_columns: + raise ValueError( + "Segments DataFrame must have the required columns. " + f"Following columns are missing: {missing_columns}" + ) + + @property + def track_ids(self) -> set[str]: + """Get track ids of tracks stored in dataset. + + Returns: + set[str]: the track ids stored. + """ + if self._segments_df.is_empty(): + return set() + return set(self._segments_df[TRACK_ID].unique().to_list()) + + @property + def offset(self) -> RelativeOffsetCoordinate: + return self._offset + + @property + def empty(self) -> bool: + """Check if the dataset is empty. + + Returns: + bool: True if the dataset is empty, False otherwise. + """ + return self._segments_df.is_empty() + + @staticmethod + def from_track_dataset( + dataset: TrackDataset, offset: RelativeOffsetCoordinate + ) -> "PolarsTrackGeometryDataset": + """Create a PolarsTrackGeometryDataset from a TrackDataset. + + Args: + dataset (TrackDataset): The source track dataset. + offset (RelativeOffsetCoordinate): Relative offset to apply to track points + before segment creation. + + Returns: + TrackGeometryDataset: A dataset containing the generated track segments. + + Raises: + ValueError: If the dataset cannot be converted to polars format. + """ + # Try to get data from pandas dataset and convert to polars + try: + if hasattr(dataset, "get_data"): + data = dataset.get_data() + # Convert pandas DataFrame to polars DataFrame + if isinstance(data, pl.DataFrame): + polars_data = data + elif hasattr(data, "to_dict"): + # It's a pandas DataFrame + polars_data = pl.DataFrame(data.to_dict(orient="list")) + else: + # Try direct conversion + polars_data = pl.DataFrame(data) + else: + raise ValueError("Cannot extract data from the provided dataset") + + segments = create_track_segments(polars_data) + return PolarsTrackGeometryDataset(offset, segments) + except Exception as e: + raise ValueError( + f"PolarsTrackGeometryDataset could not be created from dataset: {e}" + ) + + def add_all(self, tracks: Iterable[Track]) -> "PolarsTrackGeometryDataset": + """Add tracks to existing dataset. + + Pre-existing tracks will be overwritten. + + Args: + tracks (Iterable[Track]): the tracks to add. + + Returns: + PolarsTrackGeometryDataset: the dataset with tracks added. + """ + # Convert tracks to DataFrame format + track_data = self._convert_tracks_to_dataframe(tracks) + if track_data.is_empty(): + return self + + # Create segments from track data + new_segments = create_track_segments(track_data) + if new_segments.is_empty(): + return self + + if self.empty: + # If current dataset is empty, return new dataset with the new segments + return PolarsTrackGeometryDataset(self._offset, new_segments) + + # Merge with existing segments, overwriting duplicates + # Remove existing segments for tracks that are being added + new_track_ids = new_segments[TRACK_ID].unique().to_list() + existing_without_new = self._segments_df.filter( + ~pl.col(TRACK_ID).is_in(new_track_ids) + ) + + # Combine existing (without overlaps) and new segments + if not existing_without_new.is_empty() and not new_segments.is_empty(): + combined_segments = pl.concat([existing_without_new, new_segments]) + elif not existing_without_new.is_empty(): + combined_segments = existing_without_new + elif not new_segments.is_empty(): + combined_segments = new_segments + else: + combined_segments = pl.DataFrame() + + return PolarsTrackGeometryDataset(self._offset, combined_segments) + + def _convert_tracks_to_dataframe(self, tracks: Iterable[Track]) -> pl.DataFrame: + """Convert tracks to DataFrame format. + + Args: + tracks (Iterable[Track]): tracks to convert. + + Returns: + pl.DataFrame: tracks as dataframe with TRACK_ID, OCCURRENCE, X, Y, W, H. + """ + if not tracks: + return pl.DataFrame() + + data = [] + for current in tracks: + track_id = current.id.id + for detection in current.detections: + data.append( + { + TRACK_ID: track_id, + OCCURRENCE: detection.occurrence, + X: detection.x, + Y: detection.y, + W: detection.w, + H: detection.h, + } + ) + + if not data: + return pl.DataFrame() + + return pl.DataFrame(data) + + def remove(self, ids: Sequence[str]) -> "PolarsTrackGeometryDataset": + """Remove track geometries with given ids from dataset. + + Args: + ids (Sequence[str]): the track geometries to remove. + + Returns: + PolarsTrackGeometryDataset: the dataset with tracks removed. + """ + if self.empty or not ids: + return self + + # Filter out segments with track IDs in the removal list + remaining_segments = self._segments_df.filter(~pl.col(TRACK_ID).is_in(ids)) + + return PolarsTrackGeometryDataset(self._offset, remaining_segments) + + def get_for(self, track_ids: list[str]) -> "PolarsTrackGeometryDataset": + """Get geometries for given track ids if they exist. + + Ids that do not exist will not be included in the dataset. + + Args: + track_ids (list[str]): the track ids. + + Returns: + PolarsTrackGeometryDataset: the dataset with tracks. + """ + if self.empty or not track_ids: + return PolarsTrackGeometryDataset(self._offset) + + # Filter segments to include only those with track IDs in the provided list + filtered_segments = self._segments_df.filter(pl.col(TRACK_ID).is_in(track_ids)) + + return PolarsTrackGeometryDataset(self._offset, filtered_segments) + + def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + """Return a set of tracks intersecting a set of sections. + + Args: + sections (list[Section]): the list of sections to intersect. + + Returns: + set[TrackId]: the track ids intersecting the given sections. + """ + if self.empty or not sections: + return set() + + # Create a set to store the track IDs that intersect with any section + intersecting_track_ids: set[TrackId] = set() + + # Check intersections for each section + for section in sections: + # Get the coordinates of the section + coordinates = section.get_coordinates() + offset = get_section_offset(section) + + if ( + section.get_type() == SectionType.LINE + or section.get_type() == SectionType.CUTTING + ): + # For line sections, check if any track segment intersects with any leg + # of the line. A leg is formed by each consecutive pair of coordinates. + for i in range(len(coordinates) - 1): + start_x, start_y = coordinates[i].x, coordinates[i].y + end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y + + # Find intersections with this leg of the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, + offset, + ) + + # Add track IDs that intersect with the line to the result set + intersecting_segments = intersections.filter(pl.col(INTERSECTS)) + if not intersecting_segments.is_empty(): + intersecting_track_ids.update( + intersecting_segments[TRACK_ID].unique().to_list() + ) + elif section.get_type() == SectionType.AREA: + # For area sections, check if any track segment intersects with the + # polygon + polygon = Polygon(coordinates) + + # Check polygon intersections + intersections = check_polygon_intersections( + self._segments_df, polygon, offset + ) + + # Add track IDs that intersect with the polygon to the result set + intersecting_segments = intersections.filter(pl.col(INTERSECTS_POLYGON)) + if not intersecting_segments.is_empty(): + intersecting_track_ids.update( + intersecting_segments[TRACK_ID].unique().to_list() + ) + + return intersecting_track_ids + + def intersection_points( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + """Return the intersection points from tracks and the given sections. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + the intersection points. + """ + if self.empty or not sections: + return {} + + # Create a dictionary to store the intersection points for each track + result: dict[TrackId, list[tuple[SectionId, IntersectionPoint]]] = {} + + # Process only line sections + # (area sections don't have specific intersection points) + line_sections = [ + section + for section in sections + if section.get_type() == SectionType.LINE + or section.get_type() == SectionType.CUTTING + ] + + if not line_sections: + return {} + + # For each line section, find intersections with track segments + for section in line_sections: + # Get the coordinates of the section + coordinates = section.get_coordinates() + offset = get_section_offset(section) + + # Process each leg of the section (consecutive pair of coordinates) + for i in range(len(coordinates) - 1): + start_x, start_y = coordinates[i].x, coordinates[i].y + end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y + + # Find intersections with this leg of the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, + offset, + ) + + # Filter to only include segments that intersect with the line + intersecting_segments = intersections.filter(pl.col(INTERSECTS)) + + # Process each intersecting segment using polars iteration + for row in intersecting_segments.iter_rows(named=True): + track_id = row[TRACK_ID] + + # Calculate the relative position of the intersection point + # along the segment + segment_length_x = (row[END_X] + row[END_W] * offset.x) - ( + row[START_X] + row[START_W] * offset.x + ) + segment_length_y = (row[END_Y] + row[END_H] * offset.y) - ( + row[START_Y] + row[START_H] * offset.y + ) + segment_length = math.sqrt( + segment_length_x**2 + segment_length_y**2 + ) + # Avoid division by zero + if segment_length_x == 0 and segment_length_y == 0: + continue + + # Calculate the relative position (0 to 1) along the segment + intersection_length_x = row[INTERSECTION_X] - ( + row[START_X] + row[START_W] * offset.x + ) + intersection_length_y = row[INTERSECTION_Y] - ( + row[START_Y] + row[START_H] * offset.y + ) + intersection_length = math.sqrt( + intersection_length_x**2 + intersection_length_y**2 + ) + relative_position = intersection_length / segment_length + + # Create an IntersectionPoint + # The upper_index is 1 because we're dealing with segments + intersection_point = IntersectionPoint( + upper_index=1, relative_position=relative_position + ) + + # Add to the result dictionary + if track_id not in result: + result[track_id] = [] + result[track_id].append((section.id, intersection_point)) + + return result + + def contained_by_sections( + self, sections: list[Section] + ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: + raise NotImplementedError + + def __eq__(self, other: Any) -> bool: + """Check if this dataset is equal to another dataset. + + Args: + other (Any): the other dataset to compare with. + + Returns: + bool: True if the datasets are equal, False otherwise. + """ + if not isinstance(other, PolarsTrackGeometryDataset): + return False + + # If both are empty, they are equal + if self.empty and other.empty: + return True + + # If only one is empty, they are not equal + if self.empty != other.empty: + return False + + # Compare the DataFrames + # First, ensure both DataFrames have the same columns + if set(self._segments_df.columns) != set(other._segments_df.columns): + return False + + # Sort both DataFrames to ensure consistent comparison + self_df = self._segments_df.sort([TRACK_ID, START_OCCURRENCE]) + other_df = other._segments_df.sort([TRACK_ID, START_OCCURRENCE]) + + # Compare the DataFrames using polars frame_equal + try: + return self_df.equals(other_df) + except Exception: + # Fallback to row-by-row comparison if frame_equal fails + if len(self_df) != len(other_df): + return False + + # Convert to list of dicts for comparison + self_rows = self_df.to_dicts() + other_rows = other_df.to_dicts() + + return self_rows == other_rows + + def track_ids_after_cut(self, section: Section) -> pl.DataFrame: + if not section: + return pl.DataFrame() + + # Create a set to store the track IDs that intersect with any section + result = self._segments_df.select([ROW_ID, track.TRACK_ID]).with_columns( + pl.lit(False).alias(INTERSECTS) + ) + + # Get the coordinates of the section + coordinates = section.get_coordinates() + offset = get_section_offset(section) + + if section.get_type() == SectionType.CUTTING: + # For line sections, check if any track segment intersects with any leg + # of the line. A leg is formed by each consecutive pair of coordinates. + for i in range(len(coordinates) - 1): + start_x, start_y = coordinates[i].x, coordinates[i].y + end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y + + # Find intersections with this leg of the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, + offset, + ) + + # Add track IDs that intersect with the line to the result set + result = ( + result.join( + intersections.select([ROW_ID, TRACK_ID, INTERSECTS]), on=ROW_ID + ) + .with_columns( + pl.col(INTERSECTS).or_( + pl.col(f"{INTERSECTS}_right") + .fill_null(False) + .alias(INTERSECTS) + ) + ) + .select([ROW_ID, TRACK_ID, INTERSECTS]) + ) + + results = result.with_columns(pl.col(INTERSECTS).cum_sum().alias("cum_sum")) + return results.with_columns( + pl.col(TRACK_ID) + "_" + pl.col("cum_sum").cast(pl.Utf8).alias(TRACK_ID) + ).select([ROW_ID, TRACK_ID]) diff --git a/requirements.txt b/requirements.txt index 6c38fabdd..fab75ef6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ numpy==2.1.1 openpyxl==3.1.5 pandas==2.2.3 pillow==11.1.0 +polars==1.32.3 pyarrow==21.0.0 requests==2.32.3 seaborn==0.13.2 diff --git a/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py new file mode 100644 index 000000000..d74f1cf3a --- /dev/null +++ b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py @@ -0,0 +1,401 @@ +from unittest.mock import Mock, call + +import polars as pl + +from OTAnalytics.domain import track +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.plugin_datastore.polars_track_store import ( + COLUMNS, + POLARS_TRACK_GEOMETRY_FACTORY, + PolarsDetection, + PolarsTrackDataset, + PolarsTrackSegmentDataset, + _convert_tracks, + create_empty_dataframe, +) +from tests.utils.assertions import assert_equal_track_properties +from tests.utils.builders.track_builder import TrackBuilder + + +class TestPolarsDetection: + def test_properties(self) -> None: + track_id = "track_1" + data = { + track.CLASSIFICATION: "car", + track.CONFIDENCE: 0.9, + track.X: 100.0, + track.Y: 200.0, + track.W: 50.0, + track.H: 30.0, + track.FRAME: 1, + track.INTERPOLATED_DETECTION: False, + track.VIDEO_NAME: "video.mp4", + track.INPUT_FILE: "input.ottrk", + } + occurrence = TrackBuilder().create_detection().occurrence + + detection = PolarsDetection(track_id, data, occurrence) + + assert detection.track_id == TrackId(track_id) + assert detection.classification == "car" + assert detection.confidence == 0.9 + assert detection.x == 100.0 + assert detection.y == 200.0 + assert detection.w == 50.0 + assert detection.h == 30.0 + assert detection.frame == 1 + assert not detection.interpolated_detection + assert detection.video_name == "video.mp4" + assert detection.input_file == "input.ottrk" + assert detection.occurrence == occurrence + + +class TestPolarsTrack: + def test_properties(self) -> None: + track = self.create_python_track() + pandas_track = self.create_polars_track(track) + assert_equal_track_properties(pandas_track, track) + + def create_python_track(self) -> Track: + builder = TrackBuilder().add_track_id("1") + builder.append_detection() + builder.append_detection() + builder.append_detection() + return builder.build_track() + + def create_polars_track(self, track: Track) -> Track: + dataset = PolarsTrackDataset.from_list([track], Mock()) + return dataset._create_track_flyweight(track.id.id) + + +class TestPolarsTrackSegmentDataset: + def test_apply(self) -> None: + consumer = Mock() + segments = [ + {"segment_1": "value_1"}, + {"segment_2": "value_2"}, + {"segment_3": "value_3"}, + ] + df = pl.DataFrame(segments) + dataset = PolarsTrackSegmentDataset(df) + + dataset.apply(consumer) + + expected_calls = [call(segment) for segment in segments] + consumer.assert_has_calls(expected_calls) + + def test_apply_without_segments(self) -> None: + consumer = Mock() + df = pl.DataFrame() + dataset = PolarsTrackSegmentDataset(df) + + dataset.apply(consumer) + + consumer.assert_not_called() + + +class TestPolarsTrackDataset: + def _create_dataset(self, size: int) -> list[Track]: + return [self.__build_track(f"track_{i}") for i in range(size)] + + def test_use_track_classificator( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + calculator = Mock() + builder = TrackBuilder().add_track_id("1") + builder.append_detection() + builder.append_detection() + tracks = [builder.build_track()] + dataset = _convert_tracks(tracks) + calculator.calculate.return_value = pl.DataFrame( + {track.TRACK_ID: ["1"], track.TRACK_CLASSIFICATION: ["car"]} + ) + + track_dataset = PolarsTrackDataset.from_dataframe( + dataset, track_geometry_factory, calculator=calculator + ) + + assert track_dataset + calculator.calculate.assert_called_once() + + def test_add(self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY) -> None: + first_track = self.__build_track("first") + second_track = self.__build_track("second") + dataset = PolarsTrackDataset.from_list([first_track], track_geometry_factory) + + result = dataset.add_all([second_track]) + + assert len(result) == 2 + track_ids = {track_id.id for track_id in result.track_ids} + assert track_ids == {"first", "second"} + + def test_add_nothing( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + first_track = self.__build_track("first") + dataset = PolarsTrackDataset.from_list([first_track], track_geometry_factory) + + result = dataset.add_all([]) + + assert result == dataset + + def test_add_all( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + first_track = self.__build_track("first") + second_track = self.__build_track("second") + third_track = self.__build_track("third") + dataset = PolarsTrackDataset.from_list([first_track], track_geometry_factory) + + result = dataset.add_all([second_track, third_track]) + + assert len(result) == 3 + track_ids = {track_id.id for track_id in result.track_ids} + assert track_ids == {"first", "second", "third"} + + def test_add_two_existing_polars_datasets( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + first_track = self.__build_track("first") + second_track = self.__build_track("second") + first_dataset = PolarsTrackDataset.from_list( + [first_track], track_geometry_factory + ) + second_dataset = PolarsTrackDataset.from_list( + [second_track], track_geometry_factory + ) + + result = first_dataset.add_all(second_dataset) + + assert len(result) == 2 + track_ids = {track_id.id for track_id in result.track_ids} + assert track_ids == {"first", "second"} + + def test_add_all_merge_tracks( + self, + car_track: Track, + car_track_continuing: Track, + pedestrian_track: Track, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + ) -> None: + # Test that adding tracks with same ID merges them properly + dataset = PolarsTrackDataset.from_list([car_track], track_geometry_factory) + + result = dataset.add_all([car_track_continuing, pedestrian_track]) + + # Should have 2 unique tracks (car merged, pedestrian separate) + assert len(result) == 2 + track_ids = {track_id.id for track_id in result.track_ids} + expected_track_ids = {car_track.id.id, pedestrian_track.id.id} + assert track_ids == expected_track_ids + + def __build_track(self, track_id: str, length: int = 5) -> Track: + builder = TrackBuilder().add_track_id(track_id) + for i in range(length): + builder.append_detection() + return builder.build_track() + + def test_get_for( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + first_track = self.__build_track("first") + second_track = self.__build_track("second") + dataset = PolarsTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) + + result = dataset.get_for(TrackId("first")) + + assert result is not None + assert result.id == TrackId("first") + + def test_get_for_missing_id( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, car_track: Track + ) -> None: + dataset = PolarsTrackDataset.from_list([car_track], track_geometry_factory) + result = dataset.get_for(TrackId("missing")) + assert result is None + + def test_get_for_missing_id_on_empty_dataset( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + dataset = PolarsTrackDataset(track_geometry_factory) + result = dataset.get_for(TrackId("missing")) + assert result is None + + def test_clear(self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY) -> None: + first_track = self.__build_track("first") + second_track = self.__build_track("second") + dataset = PolarsTrackDataset.from_list( + [first_track, second_track], track_geometry_factory + ) + + result = dataset.clear() + + assert len(result) == 0 + assert result.empty + + def test_remove( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + first_track = self.__build_track("first") + second_track = self.__build_track("second") + third_track = self.__build_track("third") + dataset = PolarsTrackDataset.from_list( + [first_track, second_track, third_track], track_geometry_factory + ) + + result = dataset.remove(TrackId("second")) + + assert len(result) == 2 + track_ids = {track_id.id for track_id in result.track_ids} + assert track_ids == {"first", "third"} + + def test_remove_multiple( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + first_track = self.__build_track("first") + second_track = self.__build_track("second") + third_track = self.__build_track("third") + fourth_track = self.__build_track("fourth") + dataset = PolarsTrackDataset.from_list( + [first_track, second_track, third_track, fourth_track], + track_geometry_factory, + ) + + result = dataset.remove_multiple({TrackId("second"), TrackId("fourth")}) + + assert len(result) == 2 + track_ids = {track_id.id for track_id in result.track_ids} + assert track_ids == {"first", "third"} + + def test_len(self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY) -> None: + tracks = self._create_dataset(3) + dataset = PolarsTrackDataset.from_list(tracks, track_geometry_factory) + assert len(dataset) == 3 + + def test_first_occurrence( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + ) -> None: + dataset = PolarsTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + expected_first = min( + car_track.first_detection.occurrence, + pedestrian_track.first_detection.occurrence, + ) + assert dataset.first_occurrence == expected_first + + def test_last_occurrence( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + ) -> None: + dataset = PolarsTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + expected_last = max( + car_track.last_detection.occurrence, + pedestrian_track.last_detection.occurrence, + ) + assert dataset.last_occurrence == expected_last + + def test_first_occurrence_on_empty_dataset( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + dataset = PolarsTrackDataset(track_geometry_factory) + assert dataset.first_occurrence is None + + def test_last_occurrence_on_empty_dataset( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + dataset = PolarsTrackDataset(track_geometry_factory) + assert dataset.last_occurrence is None + + def test_classifications( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + ) -> None: + dataset = PolarsTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + expected = frozenset( + {car_track.classification, pedestrian_track.classification} + ) + assert dataset.classifications == expected + + def test_classifications_on_empty_dataset( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + dataset = PolarsTrackDataset(track_geometry_factory) + assert dataset.classifications == frozenset() + + def test_track_ids( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + ) -> None: + dataset = PolarsTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + expected = frozenset({car_track.id, pedestrian_track.id}) + assert dataset.track_ids == expected + + def test_empty( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, car_track: Track + ) -> None: + empty_dataset = PolarsTrackDataset(track_geometry_factory) + assert empty_dataset.empty + + non_empty_dataset = PolarsTrackDataset.from_list( + [car_track], track_geometry_factory + ) + assert not non_empty_dataset.empty + + def test_initializing_empty_dataset_has_correct_columns(self) -> None: + empty_df = create_empty_dataframe() + expected_columns = set(COLUMNS) + actual_columns = set(empty_df.columns) + assert actual_columns == expected_columns + + def test_create_test_flyweight_with_single_detection( + self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY + ) -> None: + track = self.__build_track("test_track", 1) + dataset = PolarsTrackDataset.from_list([track], track_geometry_factory) + + result = dataset.get_for(track.id) + + assert result is not None + assert result.id == track.id + assert len(result.detections) == 1 + + +def test_convert_tracks() -> None: + builder = TrackBuilder().add_track_id("1") + builder.append_detection() + builder.append_detection() + built_track = builder.build_track() + + result = _convert_tracks([built_track]) + + assert not result.is_empty() + assert len(result) == 2 # 2 detections + assert track.TRACK_ID in result.columns + assert track.OCCURRENCE in result.columns + + +def test_create_empty_dataframe() -> None: + result = create_empty_dataframe() + + assert result.is_empty() + expected_columns = set(COLUMNS) + actual_columns = set(result.columns) + assert actual_columns == expected_columns diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py new file mode 100644 index 000000000..785b6bf70 --- /dev/null +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py @@ -0,0 +1,497 @@ +""" +Test comparison between pandas and polars geometry store implementations. + +This module ensures that both pandas and polars implementations produce +identical results for all operations. +""" + +from datetime import datetime +from typing import Any, Union + +import polars as pl +import pytest +from pandas import DataFrame as PandasDataFrame +from pandas.testing import assert_frame_equal +from pytest import approx + +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.track import H, W +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + END_H, + END_OCCURRENCE, + END_W, + END_X, + END_Y, + INTERSECTION_X, + INTERSECTS, + OCCURRENCE, + START_H, + START_OCCURRENCE, + START_W, + START_X, + START_Y, + TRACK_ID, + PandasTrackGeometryDataset, +) +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + Polygon as PandasPolygon, +) +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import X, Y +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + check_polygon_intersections as pandas_check_polygon_intersections, +) +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + create_track_segments as pandas_create_track_segments, +) +from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( + find_line_intersections as pandas_find_line_intersections, +) +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + INTERSECTION_Y, + ROW_ID, + PolarsTrackGeometryDataset, +) +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + Polygon as PolarsPolygon, +) +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + check_polygon_intersections as polars_check_polygon_intersections, +) +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + create_track_segments as polars_create_track_segments, +) +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + find_line_intersections as polars_find_line_intersections, +) + + +# Test data fixtures +@pytest.fixture +def sample_track_data() -> dict[str, list[Any]]: + """Sample track data for testing.""" + return { + TRACK_ID: ["track1", "track1", "track2", "track2"], + OCCURRENCE: [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + ], + X: [0.0, 10.0, 20.0, 30.0], + Y: [0.0, 10.0, 20.0, 30.0], + W: [5.0, 5.0, 5.0, 5.0], + H: [5.0, 5.0, 5.0, 5.0], + } + + +@pytest.fixture +def sample_segments_data() -> dict[str, list[Any]]: + """Sample segments data for testing.""" + return { + TRACK_ID: ["track1", "track2"], + START_X: [60.0, 100.0], + START_Y: [0.0, 150.0], + END_X: [0.0, 110.0], + END_Y: [60.0, 160.0], + START_W: [12.0, 12.0], + START_H: [12.0, 12.0], + END_W: [12.0, 12.0], + END_H: [12.0, 12.0], + START_OCCURRENCE: [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 0), + ], + END_OCCURRENCE: [ + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 1), + ], + } + + +def dataframes_equal( + df1: Union[PandasDataFrame, pl.DataFrame], df2: Union[PandasDataFrame, pl.DataFrame] +) -> bool: + """Compare two dataframes for equality, handling pandas and polars.""" + # Convert both to pandas for comparison + if isinstance(df1, pl.DataFrame): + df1 = df1.to_pandas() + if isinstance(df2, pl.DataFrame): + df2 = df2.drop(ROW_ID, strict=False).to_pandas() + + # Handle empty dataframes + if df1.empty and df2.empty: + return True + if df1.empty or df2.empty: + return False + + # Sort by common columns for consistent comparison + sort_cols = [] + if TRACK_ID in df1.columns: + sort_cols.append(TRACK_ID) + if START_OCCURRENCE in df1.columns: + sort_cols.append(START_OCCURRENCE) + elif OCCURRENCE in df1.columns: + sort_cols.append(OCCURRENCE) + + if sort_cols: + df1 = df1.sort_values(sort_cols).reset_index(drop=True) + df2 = df2.sort_values(sort_cols).reset_index(drop=True) + + # Compare using pandas equals with appropriate tolerance for floats + try: + assert_frame_equal(df1, df2) + return True + except Exception: + try: + assert_frame_equal(df1.round(10), df2.round(10)) + return True + except Exception: + # Fallback to manual comparison + if set(df1.columns) != set(df2.columns): + return False + if len(df1) != len(df2): + return False + + for col in df1.columns: + if df1[col].dtype == "object" or df2[col].dtype == "object": + if not df1[col].equals(df2[col]): + return False + else: + try: + if not (abs(df1[col] - df2[col]) < 1e-10).all(): + return False + except Exception: + if not df1[col].equals(df2[col]): + return False + return True + + +def prepare_polars_data_frame( + data: dict[str, list[Any]], +) -> pl.DataFrame: + df = pl.DataFrame(data) + if OCCURRENCE in df.columns: + return df.with_row_index(ROW_ID).with_columns( + pl.col(OCCURRENCE).dt.cast_time_unit("ns") + ) + else: + return df.with_columns( + pl.col(START_OCCURRENCE).dt.cast_time_unit("ns") + ).with_columns(pl.col(END_OCCURRENCE).dt.cast_time_unit("ns")) + + +def prepare_polars_for_assert( + df: pl.DataFrame, +) -> pl.DataFrame: + return df.drop(ROW_ID, strict=False) + + +def prepare_pandas_for_assert( + df: PandasDataFrame, +) -> PandasDataFrame: + for column in [INTERSECTION_X, INTERSECTION_Y]: + if column in df.columns: + df[column] = df[column].astype(float) + return df + + +class TestCreateTrackSegments: + """Test create_track_segments function for both implementations.""" + + def test_create_track_segments_empty_df(self) -> None: + """Test that empty DataFrames return empty results.""" + pandas_df = PandasDataFrame() + polars_df = pl.DataFrame() + + pandas_result = pandas_create_track_segments(pandas_df) + polars_result = polars_create_track_segments(polars_df) + + assert pandas_result.empty + assert polars_result.is_empty() + + def test_create_track_segments_single_track( + self, sample_track_data: dict[str, list[Any]] + ) -> None: + """Test segment creation with single track.""" + # Filter for single track + single_track_data = {k: [v[0], v[1]] for k, v in sample_track_data.items()} + + pandas_df = PandasDataFrame(single_track_data) + polars_df = prepare_polars_data_frame(single_track_data) + + pandas_result = pandas_create_track_segments(pandas_df) + polars_result = prepare_polars_for_assert( + polars_create_track_segments(polars_df) + ) + + # Both should produce one segment + assert len(pandas_result) == 1 + assert len(polars_result) == 1 + + # Results should be equivalent + assert dataframes_equal(pandas_result, polars_result) + + def test_create_track_segments_multiple_tracks( + self, sample_track_data: dict[str, list[Any]] + ) -> None: + """Test segment creation with multiple tracks.""" + pandas_df = PandasDataFrame(sample_track_data) + polars_df = prepare_polars_data_frame(sample_track_data) + + pandas_result = pandas_create_track_segments(pandas_df) + polars_result = polars_create_track_segments(polars_df) + + # Both should produce two segments (one per track) + assert len(pandas_result) == 2 + assert len(polars_result) == 2 + + # Results should be equivalent + assert dataframes_equal(pandas_result, polars_result) + + +class TestFindLineIntersections: + """Test find_line_intersections function for both implementations.""" + + def test_find_line_intersections_empty_df(self) -> None: + """Test with empty DataFrames.""" + pandas_df = PandasDataFrame() + polars_df = pl.DataFrame() + offset = RelativeOffsetCoordinate(0.0, 0.0) + + pandas_result = pandas_find_line_intersections( + pandas_df, "line1", 0.0, 0.0, 10.0, 10.0, offset + ) + polars_result = polars_find_line_intersections( + polars_df, "line1", 0.0, 0.0, 10.0, 10.0, offset + ) + + assert pandas_result.empty + assert polars_result.is_empty() + + def test_find_line_intersections_no_intersections( + self, sample_segments_data: dict[str, list[Any]] + ) -> None: + """Test with segments that don't intersect.""" + pandas_df = PandasDataFrame(sample_segments_data) + polars_df = prepare_polars_data_frame(sample_segments_data) + offset = RelativeOffsetCoordinate(0.0, 0.0) + + # Line that doesn't intersect + line_id = "non_intersecting_line" + start_x, start_y = 0.0, 100.0 + end_x, end_y = 5.0, 105.0 + + pandas_result = pandas_find_line_intersections( + pandas_df, line_id, start_x, start_y, end_x, end_y, offset + ) + polars_result = polars_find_line_intersections( + polars_df, line_id, start_x, start_y, end_x, end_y, offset + ) + + # Both should have no intersections + assert not pandas_result[INTERSECTS].any() + assert not polars_result[INTERSECTS].any() + + # Results should be equivalent + assert dataframes_equal( + prepare_pandas_for_assert(pandas_result), + prepare_polars_for_assert(polars_result), + ) + + @pytest.mark.parametrize( + "offset, expected_intersection", + [ + (RelativeOffsetCoordinate(0.0, 0.0), 30.0), + (RelativeOffsetCoordinate(0.5, 0.0), 33.0), + (RelativeOffsetCoordinate(1.0, 0.0), 36.0), + ], + ) + def test_find_line_intersections_with_intersections( + self, + sample_segments_data: dict[str, list[Any]], + offset: RelativeOffsetCoordinate, + expected_intersection: float, + ) -> None: + """Test with segments that intersect with the line.""" + pandas_df = PandasDataFrame(sample_segments_data) + polars_df = prepare_polars_data_frame(sample_segments_data) + + # Line that intersects with first segment + line_id = "intersecting_line" + start_x, start_y = 0.0, 0.0 + end_x, end_y = 120.0, 120.0 + + pandas_result = pandas_find_line_intersections( + pandas_df, line_id, start_x, start_y, end_x, end_y, offset + ) + polars_result = polars_find_line_intersections( + polars_df, line_id, start_x, start_y, end_x, end_y, offset + ) + + # Both should have same intersection pattern + assert dataframes_equal( + prepare_pandas_for_assert(pandas_result), + prepare_polars_for_assert(polars_result), + ) + + # Check specific intersection values + pandas_intersecting = pandas_result[pandas_result[INTERSECTS]] + polars_intersecting = polars_result.to_pandas()[ + polars_result.to_pandas()[INTERSECTS] + ] + + if len(pandas_intersecting) > 0: + assert pandas_intersecting.iloc[0][INTERSECTION_X] == approx( + expected_intersection + ) + assert polars_intersecting.iloc[0][INTERSECTION_X] == approx( + expected_intersection + ) + + +class TestCheckPolygonIntersections: + """Test check_polygon_intersections function for both implementations.""" + + def test_check_polygon_intersections_empty_df(self) -> None: + """Test with empty DataFrames.""" + pandas_df = PandasDataFrame() + polars_df = pl.DataFrame() + polygon_coords = [Coordinate(0, 0), Coordinate(10, 0), Coordinate(10, 10)] + pandas_polygon = PandasPolygon(polygon_coords) + polars_polygon = PolarsPolygon(polygon_coords) + offset = RelativeOffsetCoordinate(0.0, 0.0) + + pandas_result = pandas_check_polygon_intersections( + pandas_df, pandas_polygon, offset + ) + polars_result = polars_check_polygon_intersections( + polars_df, polars_polygon, offset + ) + + assert pandas_result.empty + assert polars_result.is_empty() + + def test_check_polygon_intersections_with_intersections( + self, sample_segments_data: dict[str, list[Any]] + ) -> None: + """Test polygon intersections.""" + pandas_df = PandasDataFrame(sample_segments_data) + polars_df = prepare_polars_data_frame(sample_segments_data) + + # Create a polygon that should intersect with some segments + polygon_coords = [ + Coordinate(0, 0), + Coordinate(70, 0), + Coordinate(70, 70), + Coordinate(0, 70), + ] + pandas_polygon = PandasPolygon(polygon_coords) + polars_polygon = PolarsPolygon(polygon_coords) + offset = RelativeOffsetCoordinate(0.0, 0.0) + + pandas_result = pandas_check_polygon_intersections( + pandas_df, pandas_polygon, offset + ) + polars_result = polars_check_polygon_intersections( + polars_df, polars_polygon, offset + ) + + # Results should be equivalent + assert dataframes_equal( + prepare_pandas_for_assert(pandas_result), + prepare_polars_for_assert(polars_result), + ) + + +class TestTrackGeometryDataset: + """Test TrackGeometryDataset implementations.""" + + def test_dataset_initialization(self) -> None: + """Test dataset initialization.""" + offset = RelativeOffsetCoordinate(0.0, 0.0) + + pandas_dataset = PandasTrackGeometryDataset(offset) + polars_dataset = PolarsTrackGeometryDataset(offset) + + assert pandas_dataset.empty + assert polars_dataset.empty + assert pandas_dataset.offset == polars_dataset.offset + + def test_dataset_track_ids( + self, sample_segments_data: dict[str, list[Any]] + ) -> None: + """Test track_ids property.""" + offset = RelativeOffsetCoordinate(0.0, 0.0) + pandas_df = PandasDataFrame(sample_segments_data) + polars_df = pl.DataFrame(sample_segments_data) + + pandas_dataset = PandasTrackGeometryDataset(offset, pandas_df) + polars_dataset = PolarsTrackGeometryDataset(offset, polars_df) + + assert pandas_dataset.track_ids == polars_dataset.track_ids + + def test_dataset_get_for(self, sample_segments_data: dict[str, list[Any]]) -> None: + """Test get_for method.""" + offset = RelativeOffsetCoordinate(0.0, 0.0) + pandas_df = PandasDataFrame(sample_segments_data) + polars_df = pl.DataFrame(sample_segments_data) + + pandas_dataset = PandasTrackGeometryDataset(offset, pandas_df) + polars_dataset = PolarsTrackGeometryDataset(offset, polars_df) + + track_ids = ["track1"] + pandas_result = pandas_dataset.get_for(track_ids) + polars_result = polars_dataset.get_for(track_ids) + + assert pandas_result.track_ids == polars_result.track_ids + # Both should contain only track1 + assert pandas_result.track_ids == {"track1"} + + def test_dataset_remove(self, sample_segments_data: dict[str, list[Any]]) -> None: + """Test remove method.""" + offset = RelativeOffsetCoordinate(0.0, 0.0) + pandas_df = PandasDataFrame(sample_segments_data) + polars_df = pl.DataFrame(sample_segments_data) + + pandas_dataset = PandasTrackGeometryDataset(offset, pandas_df) + polars_dataset = PolarsTrackGeometryDataset(offset, polars_df) + + ids_to_remove = ["track1"] + pandas_result = pandas_dataset.remove(ids_to_remove) + polars_result = polars_dataset.remove(ids_to_remove) + + assert pandas_result.track_ids == polars_result.track_ids + # Both should contain only track2 + assert pandas_result.track_ids == {"track2"} + + def test_dataset_equality_empty(self) -> None: + """Test dataset equality with empty datasets.""" + offset = RelativeOffsetCoordinate(0.0, 0.0) + + pandas_dataset1 = PandasTrackGeometryDataset(offset) + pandas_dataset2 = PandasTrackGeometryDataset(offset) + polars_dataset1 = PolarsTrackGeometryDataset(offset) + polars_dataset2 = PolarsTrackGeometryDataset(offset) + + assert pandas_dataset1 == pandas_dataset2 + assert polars_dataset1 == polars_dataset2 + + def test_dataset_equality_with_data( + self, sample_segments_data: dict[str, list[Any]] + ) -> None: + """Test dataset equality with data.""" + offset = RelativeOffsetCoordinate(0.0, 0.0) + pandas_df = PandasDataFrame(sample_segments_data) + polars_df = pl.DataFrame(sample_segments_data) + + pandas_dataset1 = PandasTrackGeometryDataset(offset, pandas_df.copy()) + pandas_dataset2 = PandasTrackGeometryDataset(offset, pandas_df.copy()) + polars_dataset1 = PolarsTrackGeometryDataset(offset, polars_df.clone()) + polars_dataset2 = PolarsTrackGeometryDataset(offset, polars_df.clone()) + + assert pandas_dataset1 == pandas_dataset2 + assert polars_dataset1 == polars_dataset2 + + +if __name__ == "__main__": + pytest.main([__file__]) From b96d8116e9f231571ed84d130f2e7eeeee3319ee Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 15:47:41 +0200 Subject: [PATCH 040/128] Implement contains method --- .../pandas_geometry_store.py | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index b1d36b486..2c0b2dbfa 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -1,9 +1,15 @@ import math +from collections import defaultdict from dataclasses import dataclass from typing import Any, Iterable, Optional, Sequence +import shapely from pandas import DataFrame, concat +from shapely.creation import prepare +from shapely.geometry.base import BaseGeometry +from shapely.vectorized import contains +from OTAnalytics.domain import track from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId, SectionType @@ -17,8 +23,8 @@ from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset # Column names for track points -TRACK_ID = "track-id" -OCCURRENCE = "occurrence" +TRACK_ID = track.TRACK_ID +OCCURRENCE = track.OCCURRENCE X = "x" Y = "y" @@ -693,9 +699,9 @@ def _convert_tracks_to_dataframe(self, tracks: Iterable[Track]) -> DataFrame: return DataFrame() data = [] - for track in tracks: - track_id = track.id.id - for detection in track.detections: + for current in tracks: + track_id = current.id.id + for detection in current.detections: data.append( { TRACK_ID: track_id, @@ -916,7 +922,30 @@ def contained_by_sections( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: # TODO Do not implement - raise NotImplementedError + contains_result: dict[TrackId, list[tuple[SectionId, list[bool]]]] = ( + defaultdict(list) + ) + for _section in sections: + section_geom = area_section_to_shapely(_section) + + x_coords = self._segments_df["x"].to_numpy() + y_coords = self._segments_df["y"].to_numpy() + + contains_masks = contains(section_geom, x_coords, y_coords) + # contains_masks_old = self._segments_df[GEOMETRY].apply( + # lambda line: [ + # contains(section_geom, Point(p)) for p in get_coordinates(line) + # ] + # ) + tracks_contained = contains_masks[contains_masks.map(any)] + + if tracks_contained.empty: + continue + + tracks_contained.index = tracks_contained.index.map(TrackId) + for track_id, contains_mask in tracks_contained.to_dict().items(): + contains_result[track_id].append((_section.id, contains_mask)) + return contains_result def __eq__(self, other: Any) -> bool: """Check if this dataset is equal to another dataset. @@ -953,3 +982,9 @@ def __eq__(self, other: Any) -> bool: # Compare the DataFrames return self_df.equals(other_df) + + +def area_section_to_shapely(section: Section) -> BaseGeometry: + geometry = shapely.Polygon([(c.x, c.y) for c in section.get_coordinates()]) + prepare(geometry) + return geometry From 29426eff321c6203610350c09bd044d7dc06517d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 15:48:39 +0200 Subject: [PATCH 041/128] Use efficient conversion from polars to pandas --- .../tracks_as_dataframe_provider.py | 8 ++++++++ .../plugin_prototypes/track_visualization/track_viz.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/OTAnalytics/plugin_number_of_tracks_to_be_validated/tracks_as_dataframe_provider.py b/OTAnalytics/plugin_number_of_tracks_to_be_validated/tracks_as_dataframe_provider.py index d6eb5e51e..9763ae955 100644 --- a/OTAnalytics/plugin_number_of_tracks_to_be_validated/tracks_as_dataframe_provider.py +++ b/OTAnalytics/plugin_number_of_tracks_to_be_validated/tracks_as_dataframe_provider.py @@ -1,7 +1,9 @@ from pandas import DataFrame from OTAnalytics.application.use_cases.track_repository import GetAllTracks +from OTAnalytics.domain import track from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY +from OTAnalytics.plugin_datastore.polars_track_store import PolarsDataFrameProvider from OTAnalytics.plugin_datastore.track_store import ( DEFAULT_CLASSIFICATOR, PandasDataFrameProvider, @@ -28,6 +30,12 @@ def provide(self) -> DataFrame | None: if isinstance(track_dataset, PandasDataFrameProvider): return track_dataset.get_data() + if isinstance(track_dataset, PolarsDataFrameProvider): + return ( + track_dataset.get_data() + .to_pandas() + .set_index([track.TRACK_ID, track.OCCURRENCE]) + ) return self._convert_tracks_to_data_frame() def _convert_tracks_to_data_frame(self) -> DataFrame: diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index 567c6aee6..f3eb4d1ba 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -54,6 +54,7 @@ TrackRepository, TrackRepositoryEvent, ) +from OTAnalytics.plugin_datastore.polars_track_store import PolarsDataFrameProvider from OTAnalytics.plugin_datastore.track_store import PandasDataFrameProvider from OTAnalytics.plugin_filter.dataframe_filter import DataFrameFilterBuilder @@ -301,6 +302,12 @@ def get_data(self) -> DataFrame: tracks = self._track_repository.get_all() if isinstance(tracks, PandasDataFrameProvider): return tracks.get_data() + if isinstance(tracks, PolarsDataFrameProvider): + return ( + tracks.get_data() + .to_pandas() + .set_index([track.TRACK_ID, track.OCCURRENCE]) + ) track_list = tracks.as_list() if not track_list: return DataFrame() From f025d6f6adbfe928ebf3aa5aeeaa5078e2386594 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 15:50:11 +0200 Subject: [PATCH 042/128] Simplify shown video --- OTAnalytics/application/state.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/application/state.py b/OTAnalytics/application/state.py index 17b8138ea..d0ce89024 100644 --- a/OTAnalytics/application/state.py +++ b/OTAnalytics/application/state.py @@ -359,8 +359,9 @@ def __init__( def notify_tracks(self, track_event: TrackRepositoryEvent) -> None: all_tracks = self._datastore.get_all_tracks() if track_event.added: - first_track = next(iter(all_tracks)) - if video := self._datastore.get_video_by_date(first_track.start): + if (track_start := all_tracks.first_occurrence) and ( + video := self._datastore.get_video_by_date(track_start) + ): self._track_view_state.selected_videos.set(video) def notify_videos(self, videos: list[Video]) -> None: From d9b12088deb594aa0bffbd59c260e4c89d551006 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 15:54:35 +0200 Subject: [PATCH 043/128] Transfer from unittest to pytest and use polars instead of pandas parser --- OTAnalytics/plugin_parser/feathers_parser.py | 71 ++--- .../plugin_parser/test_feathers_parser.py | 245 ++++++++++-------- 2 files changed, 168 insertions(+), 148 deletions(-) diff --git a/OTAnalytics/plugin_parser/feathers_parser.py b/OTAnalytics/plugin_parser/feathers_parser.py index 6b28e1442..128452480 100644 --- a/OTAnalytics/plugin_parser/feathers_parser.py +++ b/OTAnalytics/plugin_parser/feathers_parser.py @@ -5,11 +5,11 @@ feather files and their accompanying metadata JSON files to create TrackParseResult. """ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Optional -import pandas as pd +import polars as pl from OTAnalytics.application.datastore import ( DetectionMetadata, @@ -18,14 +18,14 @@ TracksParseResult, ) from OTAnalytics.application.logger import logger -from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY from OTAnalytics.domain.video import VideoMetadata -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - PandasTrackGeometryDataset, +from OTAnalytics.plugin_datastore.polars_track_store import ( + POLARS_TRACK_GEOMETRY_FACTORY, + PolarsByMaxConfidence, + PolarsTrackDataset, ) -from OTAnalytics.plugin_datastore.track_store import ( - PandasByMaxConfidence, - PandasTrackDataset, +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + PolarsTrackGeometryDataset, ) from OTAnalytics.plugin_parser.convert_ottrk_to_feathers import convert_ottrk_to_feather from OTAnalytics.plugin_parser.json_parser import parse_json @@ -36,14 +36,7 @@ def use_feathers_files(files: list[Path]) -> list[Path]: result = [] for file in files: try: - if not file.suffix.lower() == ".feather": - if file.suffix.lower() == ".ottrk": - convert_ottrk_to_feather(file) - result.append(file.with_suffix(".feather")) - else: - raise ValueError( - f"Input file must have .feather or .ottrk extension: {file}" - ) + result.append(use_feather_file(file)) except Exception as cause: raised_exceptions.append(cause) if raised_exceptions: @@ -53,6 +46,19 @@ def use_feathers_files(files: list[Path]) -> list[Path]: return result +def use_feather_file(file: Path) -> Path: + if not file.suffix.lower() == ".feather": + if file.suffix.lower() == ".ottrk": + if not file.with_suffix(".feather").exists(): + convert_ottrk_to_feather(file) + return file.with_suffix(".feather") + else: + raise ValueError( + f"Input file must have .feather or .ottrk extension: {file}" + ) + return file + + class FeathersParser(TrackParser): """ Parse feather files with accompanying metadata JSON files. @@ -67,7 +73,7 @@ class FeathersParser(TrackParser): def __init__( self, - track_geometry_factory: Optional[TRACK_GEOMETRY_FACTORY] = None, + track_geometry_factory: Optional[POLARS_TRACK_GEOMETRY_FACTORY] = None, ) -> None: """ Initialize the FeathersParser. @@ -76,7 +82,7 @@ def __init__( If None, uses PandasTrackGeometryDataset.from_track_dataset. """ if track_geometry_factory is None: - track_geometry_factory = PandasTrackGeometryDataset.from_track_dataset + track_geometry_factory = PolarsTrackGeometryDataset.from_track_dataset self._track_geometry_factory = track_geometry_factory def parse_files(self, files: list[Path]) -> TracksParseResult: @@ -107,7 +113,7 @@ def parse_files(self, files: list[Path]) -> TracksParseResult: raise FileNotFoundError(f"Metadata file not found: {metadata_file}") # Read the feather file - df = pd.read_feather(file) + df = pl.read_ipc(file) data_frames.append(df) # Read the metadata @@ -124,10 +130,10 @@ def parse_files(self, files: list[Path]) -> TracksParseResult: detections_metadata.append(detection_metadata) logger().info(f"{len(files)} track files parsed.") - df = pd.concat(data_frames) + df = pl.concat(data_frames) # Create TrackDataset from DataFrame - calculator = PandasByMaxConfidence() - tracks = PandasTrackDataset.from_dataframe( + calculator = PolarsByMaxConfidence() + tracks = PolarsTrackDataset.from_dataframe( df, self._track_geometry_factory, calculator=calculator ) logger().info("TrackDataset created.") @@ -147,12 +153,7 @@ def parse(self, file: Path) -> TrackParseResult: FileNotFoundError: If the feather file or metadata file is not found ValueError: If the file extension is not .feather """ - - if not file.suffix.lower() == ".feather": - if file.suffix.lower() == ".ottrk": - file = file.with_suffix(".feather") - else: - raise ValueError(f"Input file must have .feather extension: {file}") + file = use_feather_file(file) if not file.exists(): raise FileNotFoundError(f"Feather file not found: {file}") @@ -162,14 +163,14 @@ def parse(self, file: Path) -> TrackParseResult: raise FileNotFoundError(f"Metadata file not found: {metadata_file}") # Read the feather file - df = pd.read_feather(file) + df = pl.read_ipc(file) # Read the metadata metadata = parse_json(metadata_file) # Create TrackDataset from DataFrame - calculator = PandasByMaxConfidence() - tracks = PandasTrackDataset.from_dataframe( + calculator = PolarsByMaxConfidence() + tracks = PolarsTrackDataset.from_dataframe( df, self._track_geometry_factory, calculator=calculator ) @@ -193,12 +194,14 @@ def _parse_video_metadata(self, metadata: dict) -> VideoMetadata: Returns: VideoMetadata: Parsed video metadata object """ - recorded_start_date = datetime.fromisoformat(metadata["recorded_start_date"]) + recorded_start_date = datetime.fromtimestamp( + metadata["recorded_start_date"], tz=timezone.utc + ) # Parse optional fields expected_duration = None - if "expected_duration" in metadata: - expected_duration = timedelta(seconds=metadata["expected_duration"]) + if entry := metadata.get("expected_duration"): + expected_duration = timedelta(seconds=entry) actual_fps = metadata.get("actual_fps") diff --git a/tests/OTAnalytics/plugin_parser/test_feathers_parser.py b/tests/OTAnalytics/plugin_parser/test_feathers_parser.py index f7ce392a9..a44e73aa2 100644 --- a/tests/OTAnalytics/plugin_parser/test_feathers_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_feathers_parser.py @@ -5,7 +5,7 @@ import tempfile from datetime import datetime, timedelta from pathlib import Path -from unittest import TestCase +from typing import Any from unittest.mock import Mock, patch import pandas as pd @@ -13,95 +13,103 @@ from OTAnalytics.application.datastore import DetectionMetadata, TrackParseResult from OTAnalytics.domain.video import VideoMetadata -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - PandasTrackGeometryDataset, +from OTAnalytics.plugin_datastore.polars_track_store import PolarsTrackDataset +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + PolarsTrackGeometryDataset, ) -from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset from OTAnalytics.plugin_parser.feathers_parser import FeathersParser -class TestFeathersParser(TestCase): - - def setUp(self) -> None: - """Set up test fixtures.""" - self.parser = FeathersParser() - - # Sample DataFrame for testing - self.sample_df = pd.DataFrame( - { - "track_id": ["1", "1", "2", "2"], - "frame": [1, 2, 1, 2], - "x": [100.0, 105.0, 200.0, 205.0], - "y": [100.0, 105.0, 200.0, 205.0], - "w": [50.0, 50.0, 60.0, 60.0], - "h": [80.0, 80.0, 90.0, 90.0], - "classification": ["car", "car", "truck", "truck"], - "confidence": [0.9, 0.85, 0.92, 0.88], - "occurrence": [ - datetime(2023, 1, 1, 10, 0, 0), - datetime(2023, 1, 1, 10, 0, 1), - datetime(2023, 1, 1, 10, 0, 0), - datetime(2023, 1, 1, 10, 0, 1), - ], - "video_name": [ - "test_video.mp4", - "test_video.mp4", - "test_video.mp4", - "test_video.mp4", - ], - "input_file": ["test.ottrk", "test.ottrk", "test.ottrk", "test.ottrk"], - "interpolated_detection": [False, False, False, False], - "original_track_id": ["1", "1", "2", "2"], - } - ) - - # Sample metadata for testing - self.sample_metadata = { - "detection_metadata": {"detection_classes": ["car", "truck", "bus"]}, - "video_metadata": { - "path": "test_video.mp4", - "recorded_start_date": "2023-01-01T10:00:00+00:00", - "recorded_fps": 30.0, - "number_of_frames": 900, - "expected_duration": 30.0, - "actual_fps": 29.97, - }, +@pytest.fixture +def parser() -> FeathersParser: + """Create a FeathersParser instance for testing.""" + return FeathersParser() + + +@pytest.fixture +def sample_df() -> pd.DataFrame: + """Create sample DataFrame for testing.""" + return pd.DataFrame( + { + "track_id": ["1", "1", "2", "2"], + "frame": [1, 2, 1, 2], + "x": [100.0, 105.0, 200.0, 205.0], + "y": [100.0, 105.0, 200.0, 205.0], + "w": [50.0, 50.0, 60.0, 60.0], + "h": [80.0, 80.0, 90.0, 90.0], + "classification": ["car", "car", "truck", "truck"], + "confidence": [0.9, 0.85, 0.92, 0.88], + "occurrence": [ + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + datetime(2023, 1, 1, 10, 0, 0), + datetime(2023, 1, 1, 10, 0, 1), + ], + "video_name": [ + "test_video.mp4", + "test_video.mp4", + "test_video.mp4", + "test_video.mp4", + ], + "input_file": ["test.ottrk", "test.ottrk", "test.ottrk", "test.ottrk"], + "interpolated_detection": [False, False, False, False], + "original_track_id": ["1", "1", "2", "2"], } + ) + + +@pytest.fixture +def sample_metadata() -> dict[str, Any]: + """Create sample metadata for testing.""" + return { + "detection_metadata": {"detection_classes": ["car", "truck", "bus"]}, + "video_metadata": { + "path": "test_video.mp4", + "recorded_start_date": 1672574400.0, # 2023-01-01T10:00:00 as timestamp + "recorded_fps": 30.0, + "number_of_frames": 900, + "expected_duration": 30.0, + "actual_fps": 29.97, + }, + } + + +class TestFeathersParser: def test_init_default_geometry_factory(self) -> None: """Test initialization with default geometry factory.""" parser = FeathersParser() - self.assertEqual( - parser._track_geometry_factory, - PandasTrackGeometryDataset.from_track_dataset, + assert ( + parser._track_geometry_factory + == PolarsTrackGeometryDataset.from_track_dataset ) def test_init_custom_geometry_factory(self) -> None: """Test initialization with custom geometry factory.""" custom_factory = Mock() parser = FeathersParser(track_geometry_factory=custom_factory) - self.assertEqual(parser._track_geometry_factory, custom_factory) + assert parser._track_geometry_factory == custom_factory - def test_parse_file_not_found(self) -> None: + def test_parse_file_not_found(self, parser: FeathersParser) -> None: """Test parsing when feather file doesn't exist.""" non_existent_file = Path("/non/existent/file.feather") - with self.assertRaises(FileNotFoundError) as context: - self.parser.parse(non_existent_file) + with pytest.raises(FileNotFoundError) as exc_info: + parser.parse(non_existent_file) - self.assertIn("Feather file not found", str(context.exception)) + assert "Feather file not found" in str(exc_info.value) - def test_parse_wrong_extension(self) -> None: + def test_parse_wrong_extension(self, parser: FeathersParser) -> None: """Test parsing file with wrong extension.""" with tempfile.NamedTemporaryFile(suffix=".txt") as temp_file: temp_path = Path(temp_file.name) - with self.assertRaises(ValueError) as context: - self.parser.parse(temp_path) + with pytest.raises(ValueError) as exc_info: + parser.parse(temp_path) - self.assertIn("must have .feather extension", str(context.exception)) + assert "must have .feather or .ottrk extension" in str(exc_info.value) - def test_parse_metadata_file_not_found(self) -> None: + def test_parse_metadata_file_not_found(self, parser: FeathersParser) -> None: """Test parsing when metadata file doesn't exist.""" with tempfile.NamedTemporaryFile(suffix=".feather", delete=False) as temp_file: temp_path = Path(temp_file.name) @@ -109,24 +117,33 @@ def test_parse_metadata_file_not_found(self) -> None: pd.DataFrame().to_feather(temp_file.name) try: - with self.assertRaises(FileNotFoundError) as context: - self.parser.parse(temp_path) + with pytest.raises(FileNotFoundError) as exc_info: + parser.parse(temp_path) - self.assertIn("Metadata file not found", str(context.exception)) + assert "Metadata file not found" in str(exc_info.value) finally: temp_path.unlink() # Clean up @patch("OTAnalytics.plugin_parser.feathers_parser.parse_json") - @patch("pandas.read_feather") - @patch("OTAnalytics.plugin_datastore.track_store.PandasTrackDataset.from_dataframe") + @patch("polars.read_ipc") + @patch( + "OTAnalytics.plugin_datastore.polars_track_store." + "PolarsTrackDataset.from_dataframe" + ) def test_parse_success( - self, mock_from_dataframe: Mock, mock_read_feather: Mock, mock_parse_json: Mock + self, + mock_from_dataframe: Mock, + mock_read_ipc: Mock, + mock_parse_json: Mock, + parser: FeathersParser, + sample_df: pd.DataFrame, + sample_metadata: dict[str, Any], ) -> None: """Test successful parsing of feather file and metadata.""" # Set up mocks - mock_read_feather.return_value = self.sample_df - mock_parse_json.return_value = self.sample_metadata - mock_track_dataset = Mock(spec=PandasTrackDataset) + mock_read_ipc.return_value = sample_df + mock_parse_json.return_value = sample_metadata + mock_track_dataset = Mock(spec=PolarsTrackDataset) mock_from_dataframe.return_value = mock_track_dataset with tempfile.NamedTemporaryFile( @@ -138,29 +155,29 @@ def test_parse_success( metadata_path.write_text('{"test": "data"}') try: - result = self.parser.parse(feather_path) + result = parser.parse(feather_path) # Verify result type and structure - self.assertIsInstance(result, TrackParseResult) - self.assertEqual(result.tracks, mock_track_dataset) + assert isinstance(result, TrackParseResult) + assert result.tracks == mock_track_dataset # Verify video metadata video_metadata = result.video_metadata - self.assertIsInstance(video_metadata, VideoMetadata) - self.assertEqual(video_metadata.path, "test_video.mp4") - self.assertEqual(video_metadata.recorded_fps, 30.0) - self.assertEqual(video_metadata.number_of_frames, 900) - self.assertEqual(video_metadata.actual_fps, 29.97) - self.assertEqual(video_metadata.expected_duration, timedelta(seconds=30.0)) + assert isinstance(video_metadata, VideoMetadata) + assert video_metadata.path == "test_video.mp4" + assert video_metadata.recorded_fps == 30.0 + assert video_metadata.number_of_frames == 900 + assert video_metadata.actual_fps == 29.97 + assert video_metadata.expected_duration == timedelta(seconds=30.0) # Verify detection metadata detection_metadata = result.detection_metadata - self.assertIsInstance(detection_metadata, DetectionMetadata) + assert isinstance(detection_metadata, DetectionMetadata) expected_classes = frozenset(["car", "truck", "bus"]) - self.assertEqual(detection_metadata.detection_classes, expected_classes) + assert detection_metadata.detection_classes == expected_classes # Verify mock calls - mock_read_feather.assert_called_once_with(feather_path) + mock_read_ipc.assert_called_once_with(feather_path) mock_parse_json.assert_called_once_with(metadata_path) mock_from_dataframe.assert_called_once() @@ -170,65 +187,65 @@ def test_parse_success( if metadata_path.exists(): metadata_path.unlink() - def test_parse_video_metadata_minimal(self) -> None: + def test_parse_video_metadata_minimal(self, parser: FeathersParser) -> None: """Test parsing video metadata with minimal required fields.""" minimal_metadata = { "path": "video.mp4", - "recorded_start_date": "2023-01-01T10:00:00+00:00", + "recorded_start_date": 1672574400.0, # 2023-01-01T10:00:00 as timestamp "recorded_fps": 30.0, "number_of_frames": 900, } - result = self.parser._parse_video_metadata(minimal_metadata) + result = parser._parse_video_metadata(minimal_metadata) - self.assertIsInstance(result, VideoMetadata) - self.assertEqual(result.path, "video.mp4") - self.assertEqual(result.recorded_fps, 30.0) - self.assertEqual(result.number_of_frames, 900) - self.assertIsNone(result.expected_duration) - self.assertIsNone(result.actual_fps) + assert isinstance(result, VideoMetadata) + assert result.path == "video.mp4" + assert result.recorded_fps == 30.0 + assert result.number_of_frames == 900 + assert result.expected_duration is None + assert result.actual_fps is None - def test_parse_video_metadata_with_optional_fields(self) -> None: + def test_parse_video_metadata_with_optional_fields( + self, parser: FeathersParser + ) -> None: """Test parsing video metadata with all optional fields.""" full_metadata = { "path": "video.mp4", - "recorded_start_date": "2023-01-01T10:00:00+00:00", + "recorded_start_date": 1672574400.0, # 2023-01-01T10:00:00 as timestamp "recorded_fps": 30.0, "number_of_frames": 900, "expected_duration": 30.0, "actual_fps": 29.97, } - result = self.parser._parse_video_metadata(full_metadata) + result = parser._parse_video_metadata(full_metadata) - self.assertIsInstance(result, VideoMetadata) - self.assertEqual(result.path, "video.mp4") - self.assertEqual(result.recorded_fps, 30.0) - self.assertEqual(result.number_of_frames, 900) - self.assertEqual(result.expected_duration, timedelta(seconds=30.0)) - self.assertEqual(result.actual_fps, 29.97) + assert isinstance(result, VideoMetadata) + assert result.path == "video.mp4" + assert result.recorded_fps == 30.0 + assert result.number_of_frames == 900 + assert result.expected_duration == timedelta(seconds=30.0) + assert result.actual_fps == 29.97 - def test_parse_detection_metadata(self) -> None: + def test_parse_detection_metadata(self, parser: FeathersParser) -> None: """Test parsing detection metadata.""" detection_metadata = { "detection_classes": ["car", "truck", "bus", "motorcycle"] } - result = self.parser._parse_detection_metadata(detection_metadata) + result = parser._parse_detection_metadata(detection_metadata) - self.assertIsInstance(result, DetectionMetadata) + assert isinstance(result, DetectionMetadata) expected_classes = frozenset(["car", "truck", "bus", "motorcycle"]) - self.assertEqual(result.detection_classes, expected_classes) + assert result.detection_classes == expected_classes - def test_parse_detection_metadata_empty_classes(self) -> None: + def test_parse_detection_metadata_empty_classes( + self, parser: FeathersParser + ) -> None: """Test parsing detection metadata with empty classes list.""" detection_metadata: dict[str, list] = {"detection_classes": []} - result = self.parser._parse_detection_metadata(detection_metadata) - - self.assertIsInstance(result, DetectionMetadata) - self.assertEqual(result.detection_classes, frozenset()) - + result = parser._parse_detection_metadata(detection_metadata) -if __name__ == "__main__": - pytest.main([__file__]) + assert isinstance(result, DetectionMetadata) + assert result.detection_classes == frozenset() From 2965f3cdb62e0d344b2f36a778b10a0814481ef6 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 15:56:12 +0200 Subject: [PATCH 044/128] Move calculation of ids inside section into dataset --- .../use_cases/inside_cutting_section.py | 32 ++----------------- .../domain/track_dataset/track_dataset.py | 25 +++++++++++++++ 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/OTAnalytics/application/use_cases/inside_cutting_section.py b/OTAnalytics/application/use_cases/inside_cutting_section.py index fa0e4bbca..c48bf0d5e 100644 --- a/OTAnalytics/application/use_cases/inside_cutting_section.py +++ b/OTAnalytics/application/use_cases/inside_cutting_section.py @@ -2,14 +2,9 @@ from OTAnalytics.application.use_cases.section_repository import GetCuttingSections from OTAnalytics.application.use_cases.track_repository import GetAllTracks -from OTAnalytics.domain.section import ( - SectionId, - SectionListObserver, - SectionRepositoryEvent, -) +from OTAnalytics.domain.section import SectionListObserver, SectionRepositoryEvent from OTAnalytics.domain.track import TrackId, TrackIdProvider from OTAnalytics.domain.track_repository import TrackListObserver, TrackRepositoryEvent -from OTAnalytics.domain.types import EventType class TrackIdsInsideCuttingSections(TrackIdProvider): @@ -25,23 +20,7 @@ def get_ids(self) -> set[TrackId]: if not cutting_sections: return set() - results: set[TrackId] = set() - for cutting_section in cutting_sections: - offset = cutting_section.get_offset(EventType.SECTION_ENTER) - # set of all tracks where at least one coordinate is contained - # by at least one cutting section - results.update( - set( - track_id - for track_id, section_data in ( - track_dataset.contained_by_sections( - [cutting_section], offset - ).items() - ) - if contains_true(section_data) - ) - ) - return results + return track_dataset.ids_inside(cutting_sections) class CachedTrackIdsInsideCuttingSections( @@ -79,10 +58,3 @@ def _remove_track_ids(self, remove_track_ids: Iterable[TrackId]) -> None: def __get_empty_cache(self) -> set[TrackId]: return set() - - -def contains_true(section_data: list[tuple[SectionId, list[bool]]]) -> bool: - for _, bool_list in section_data: - if any(bool_list): - return True - return False diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 0bb64939f..58e0d20a1 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -6,6 +6,7 @@ from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.types import EventType START_X: str = "start_x" START_Y: str = "start_y" @@ -23,6 +24,13 @@ class TrackDoesNotExistError(Exception): pass +def contains_true(section_data: list[tuple[SectionId, list[bool]]]) -> bool: + for _, bool_list in section_data: + if any(bool_list): + return True + return False + + @dataclass(frozen=True, order=True) class IntersectionPoint: upper_index: int @@ -253,6 +261,23 @@ def revert_cuts_for(self, original_track_ids: frozenset[TrackId]) -> "TrackDatas """ raise NotImplementedError + def ids_inside(self, sections: list[Section]) -> set[TrackId]: + results: set[TrackId] = set() + for cutting_section in sections: + offset = cutting_section.get_offset(EventType.SECTION_ENTER) + # set of all tracks where at least one coordinate is contained + # by at least one cutting section + results.update( + set( + track_id + for track_id, section_data in ( + self.contained_by_sections([cutting_section], offset).items() + ) + if contains_true(section_data) + ) + ) + return results + class TrackGeometryDataset(ABC): """Dataset containing track geometries. From eaedc36b22208fc16db9e17d2630d54960217ace Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 15:57:02 +0200 Subject: [PATCH 045/128] Fix different datatypes of ids --- .../application/use_cases/track_repository.py | 6 ++++-- OTAnalytics/plugin_datastore/track_store.py | 17 ++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index 67abbb4b6..4979c1700 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -2,7 +2,7 @@ from typing import Iterable from OTAnalytics.application.logger import logger -from OTAnalytics.domain.track import Track, TrackId, TrackIdProvider +from OTAnalytics.domain.track import Track, TrackId, TrackIdProvider, pack, unpack from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset from OTAnalytics.domain.track_repository import ( RemoveMultipleTracksError, @@ -25,7 +25,9 @@ def __init__(self, other: TrackIdProvider, by: TrackIdProvider): self._by = by def get_ids(self) -> set[TrackId]: - return set(self._other.get_ids()).intersection(self._by.get_ids()) + other_ids = set({unpack(id) for id in self._other.get_ids()}) + self_ids = set({unpack(id) for id in self._by.get_ids()}) + return {pack(id) for id in other_ids.intersection(self_ids)} class GetAllTracks: diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 1006ba71f..b41dbdff4 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -15,7 +15,7 @@ from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track import Detection, Track, TrackId, unpack from OTAnalytics.domain.track_dataset.filtered_track_dataset import ( FilterByClassTrackDataset, FilteredTrackDataset, @@ -248,14 +248,6 @@ def get_data(self) -> DataFrame: pass -def unpack(track_id: TrackId | str) -> str: - return track_id.id if isinstance(track_id, TrackId) else track_id - - -def pack(track_id: TrackId | str) -> TrackId: - return track_id if isinstance(track_id, TrackId) else TrackId(track_id) - - class PandasTrackDataset(TrackDataset, PandasDataFrameProvider): @property def track_ids(self) -> frozenset[TrackId]: @@ -608,6 +600,13 @@ def cut_with_section( if len(self) == 0: logger().info("No tracks to cut") return self, set() + + # TODO is this only: + # - groupby trackid + # - shift by 1 + # - calculate intersects per segment + # - cumcount per intersects by track id + # - add cumcount to track id intersection_points = self.intersection_points([section], offset) cut_indices = { unpack(track_id): [ From 21e7a45e81b04ce4f5ca55d745cacf3e6658fbe4 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 16:02:39 +0200 Subject: [PATCH 046/128] Use polars track dataset with simplified cutting --- .../simple/cut_tracks_with_sections.py | 16 +---- OTAnalytics/plugin_ui/base_application.py | 70 ++++++------------- .../simple/test_cut_tracks_with_sections.py | 20 +----- tests/OTAnalytics/plugin_ui/test_cli.py | 5 -- tests/benchmark_otanalytics.py | 52 +++++++++----- 5 files changed, 59 insertions(+), 104 deletions(-) diff --git a/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py b/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py index 15c6fb480..cf8988328 100644 --- a/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py +++ b/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py @@ -8,11 +8,7 @@ GetSectionsById, RemoveSection, ) -from OTAnalytics.application.use_cases.track_repository import ( - AddAllTracks, - GetAllTracks, - RemoveTracks, -) +from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.observer import OBSERVER, Subject from OTAnalytics.domain.section import ( Section, @@ -29,10 +25,6 @@ class SimpleCutTracksIntersectingSection(CutTracksIntersectingSection): Args: get_sections_by_id (GetSectionsById): get sections by id. get_tracks (GetAllTracks): get all tracks. - add_all_tracks (AddAllTracks): used to add all tracks to the track - repository. - remove_tracks (RemoveTracks): used to remove original tracks that have been - cut. remove_section (RemoveSection): used to remove the cutting section. """ @@ -40,16 +32,12 @@ def __init__( self, get_sections_by_id: GetSectionsById, get_tracks: GetAllTracks, - add_all_tracks: AddAllTracks, - remove_tracks: RemoveTracks, remove_section: RemoveSection, ) -> None: self._subject: Subject[CutTracksDto] = Subject[CutTracksDto]() self._get_sections_by_id = get_sections_by_id self._get_tracks = get_tracks - self._add_all_tracks = add_all_tracks - self._remove_tracks = remove_tracks self._remove_section = remove_section def __call__( @@ -59,8 +47,6 @@ def __call__( cut_tracks_dataset, ids_of_cut_tracks = track_dataset.cut_with_section( cutting_section, cutting_section.get_offset(EventType.SECTION_ENTER) ) - self._remove_tracks(ids_of_cut_tracks) - self._add_all_tracks(cut_tracks_dataset) if not preserve_cutting_section: self._remove_section(cutting_section.id) self._subject.notify( diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index 242f67ded..b13ef9aa7 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -173,21 +173,16 @@ from OTAnalytics.domain.remark import RemarkRepository from OTAnalytics.domain.section import SectionRepository from OTAnalytics.domain.track import TrackIdProvider -from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository -from OTAnalytics.plugin_datastore.pandas_track_dataset_factory import ( - PandasTrackDatasetFactory, - TypeCheckingPandasTrackDatasetFactory, +from OTAnalytics.plugin_datastore.polars_track_store import ( + POLARS_TRACK_GEOMETRY_FACTORY, + PolarsByMaxConfidence, + PolarsTrackDataset, ) from OTAnalytics.plugin_datastore.python_track_store import ByMaxConfidence -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - PandasTrackGeometryDataset, -) -from OTAnalytics.plugin_datastore.track_store import ( - FilterByClassPandasTrackDataset, - PandasByMaxConfidence, - PandasTrackDataset, +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + PolarsTrackGeometryDataset, ) from OTAnalytics.plugin_intersect.simple.cut_tracks_with_sections import ( SimpleCutTracksIntersectingSection, @@ -233,10 +228,8 @@ OtEventListParser, OtFlowParser, OttrkFormatFixer, - OttrkParser, SimpleVideoParser, ) -from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser from OTAnalytics.plugin_parser.road_user_assignment_export import ( SimpleRoadUserAssignmentExporterFactory, ) @@ -581,30 +574,19 @@ def datastore(self) -> Datastore: @cached_property def track_repository(self) -> TrackRepository: return TrackRepository( - FilterByClassPandasTrackDataset( - PandasTrackDataset.from_list( - [], - self.track_geometry_factory, - self.pandas_by_max_confidence, - ), - self.run_config.include_classes, - self.run_config.exclude_classes, - ) + # FilterByClassPandasTrackDataset( + PolarsTrackDataset.from_list( + [], + self.track_geometry_factory, + self.polars_by_max_confidence, + ), + # self.run_config.include_classes, + # self.run_config.exclude_classes, + # ) ) def _create_track_parser(self) -> TrackParser: - return self._create_feathers_parser() - - def _create_ottrk_parser(self) -> TrackParser: - detection_parser = PandasDetectionParser( - self.pandas_by_max_confidence, - self.track_geometry_factory, - track_length_limit=DEFAULT_TRACK_LENGTH_LIMIT, - ) - return OttrkParser(detection_parser) - - def _create_feathers_parser(self) -> TrackParser: - return FeathersParser() + return FeathersParser(self.track_geometry_factory) def _create_stream_track_parser(self) -> StreamTrackParser: return StreamOttrkParser( @@ -614,10 +596,10 @@ def _create_stream_track_parser(self) -> StreamTrackParser: ), format_fixer=OttrkFormatFixer(), progressbar=TqdmBuilder(), - track_dataset_factory=lambda tracks: PandasTrackDataset.from_list( + track_dataset_factory=lambda tracks: PolarsTrackDataset.from_list( tracks, self.track_geometry_factory, - self.pandas_by_max_confidence, + self.polars_by_max_confidence, ), chunk_size=self.run_config.cli_chunk_size, ) @@ -816,8 +798,6 @@ def cut_tracks_intersecting_section(self) -> CutTracksIntersectingSection: return SimpleCutTracksIntersectingSection( self.get_sections_by_id, self.get_all_tracks, - self.add_all_tracks, - self.remove_tracks, self.remove_section, ) @@ -973,18 +953,12 @@ def resource_manager(self) -> ResourceManager: return ResourceManager() @cached_property - def pandas_track_dataset_factory(self) -> PandasTrackDatasetFactory: - return TypeCheckingPandasTrackDatasetFactory( - self.track_geometry_factory, self.pandas_by_max_confidence - ) - - @cached_property - def pandas_by_max_confidence(self) -> PandasByMaxConfidence: - return PandasByMaxConfidence() + def polars_by_max_confidence(self) -> PolarsByMaxConfidence: + return PolarsByMaxConfidence() @cached_property - def track_geometry_factory(self) -> TRACK_GEOMETRY_FACTORY: - return PandasTrackGeometryDataset.from_track_dataset + def track_geometry_factory(self) -> POLARS_TRACK_GEOMETRY_FACTORY: + return PolarsTrackGeometryDataset.from_track_dataset def create_format_fixer( diff --git a/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py b/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py index 55e8b8fa2..e2dc36e03 100644 --- a/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py +++ b/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, call, patch +from unittest.mock import Mock, patch import pytest @@ -7,9 +7,7 @@ RemoveSection, ) from OTAnalytics.application.use_cases.track_repository import ( - AddAllTracks, GetTracksWithoutSingleDetections, - RemoveTracks, ) from OTAnalytics.domain.section import ( Area, @@ -59,15 +57,11 @@ def test_cut(self, cutting_section: LineSection) -> None: track_dataset.cut_with_section.return_value = (cut_tracks_dataset, {track_id}) get_tracks.as_dataset.return_value = track_dataset - add_all_tracks = Mock(spec=AddAllTracks) - remove_tracks = Mock(spec=RemoveTracks) remove_section = Mock(spec=RemoveSection) cut_tracks_intersecting_section = SimpleCutTracksIntersectingSection( get_sections_by_id, get_tracks, - add_all_tracks, - remove_tracks, remove_section, ) cut_tracks_intersecting_section(cutting_section, False) @@ -75,14 +69,6 @@ def test_cut(self, cutting_section: LineSection) -> None: assert get_tracks.as_dataset.call_count == 2 - assert add_all_tracks.call_args_list == [ - call(cut_tracks_dataset), - call(cut_tracks_dataset), - ] - assert remove_tracks.call_args_list == [ - call({track_id}), - call({track_id}), - ] remove_section.assert_called_once_with(cutting_section.id) def test_notify_sections( @@ -97,7 +83,7 @@ def test_notify_sections( with patch.object(SimpleCutTracksIntersectingSection, "__call__") as call_mock: cut_tracks_intersecting_section = SimpleCutTracksIntersectingSection( - get_sections_by_id, Mock(), Mock(), Mock(), Mock() + get_sections_by_id, Mock(), Mock() ) cut_tracks_intersecting_section.notify_sections( SectionRepositoryEvent.create_added(section_ids) @@ -112,7 +98,7 @@ def test_notify_sections( def test_register(self, mock_subject_register: Mock) -> None: observer = Mock() cut_tracks_intersecting_section = SimpleCutTracksIntersectingSection( - Mock(), Mock(), Mock(), Mock(), Mock() + Mock(), Mock(), Mock() ) cut_tracks_intersecting_section.register(observer) mock_subject_register.assert_called_once_with(observer) diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index ed7c7e18f..3d68a3809 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -79,7 +79,6 @@ GetAllTrackIds, GetAllTracks, GetTracksWithoutSingleDetections, - RemoveTracks, TrackRepositorySize, ) from OTAnalytics.application.use_cases.track_statistics import CalculateTrackStatistics @@ -417,8 +416,6 @@ def cli_dependencies_old(self) -> dict[str, Any]: cut_tracks = SimpleCutTracksIntersectingSection( GetSectionsById(section_repository), get_all_tracks, - add_all_tracks, - RemoveTracks(track_repository), RemoveSection(section_repository), ) apply_cli_cuts = ApplyCliCuts(cut_tracks, TrackRepositorySize(track_repository)) @@ -512,8 +509,6 @@ def cli_dependencies(self) -> dict[str, Any]: cut_tracks = SimpleCutTracksIntersectingSection( GetSectionsById(section_repository), get_all_tracks, - add_all_tracks, - RemoveTracks(track_repository), RemoveSection(section_repository), ) apply_cli_cuts = ApplyCliCuts(cut_tracks, TrackRepositorySize(track_repository)) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index 0ea7b949f..a093fbd66 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -29,10 +29,8 @@ RemoveSection, ) from OTAnalytics.application.use_cases.track_repository import ( - AddAllTracks, GetAllTracks, GetTracksWithoutSingleDetections, - RemoveTracks, ) from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.flow import FlowRepository @@ -51,13 +49,17 @@ from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_cli.cli_application import OtAnalyticsCliApplicationStarter +from OTAnalytics.plugin_datastore.polars_track_store import ( + POLARS_TRACK_GEOMETRY_FACTORY, + PolarsTrackDataset, +) from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, FilteredPythonTrackDataset, PythonTrackDataset, ) -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - PandasTrackGeometryDataset, +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + PolarsTrackGeometryDataset, ) from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, @@ -70,6 +72,7 @@ from OTAnalytics.plugin_intersect.simple.cut_tracks_with_sections import ( SimpleCutTracksIntersectingSection, ) +from OTAnalytics.plugin_parser.feathers_parser import FeathersParser from OTAnalytics.plugin_parser.otvision_parser import ( OtFlowParser, OttrkParser, @@ -81,7 +84,8 @@ PYTHON = "PYTHON" PANDAS = "PANDAS" PURE_PANDAS = "PURE_PANDAS" -CURRENT_DATASET_TYPE = PURE_PANDAS +POLARS = "POLARS" +CURRENT_DATASET_TYPE = POLARS EXCLUDE_FILTER = [ OtcClasses.PEDESTRIAN, @@ -118,10 +122,10 @@ def __init__( self._dataset_type = dataset_type @cached_property - def track_geometry_factory(self) -> TRACK_GEOMETRY_FACTORY: - if self._dataset_type == PURE_PANDAS: - return PandasTrackGeometryDataset.from_track_dataset - return ShapelyTrackGeometryDataset.from_track_dataset + def track_geometry_factory(self) -> POLARS_TRACK_GEOMETRY_FACTORY: + if self._dataset_type == POLARS: + return PolarsTrackGeometryDataset.from_track_dataset + raise ValueError(f"Unknown dataset type {self._dataset_type}") class UseCaseProvider: @@ -201,14 +205,10 @@ def get_export_counts(self) -> ExportCounts: def get_cut_tracks(self) -> CutTracksIntersectingSection: get_sections_by_id = GetSectionsById(self._section_repository) get_tracks = GetAllTracks(self._track_repository) - add_all_tracks = AddAllTracks(self._track_repository) - remove_tracks = RemoveTracks(self._track_repository) remove_section = RemoveSection(self._section_repository) return SimpleCutTracksIntersectingSection( get_sections_by_id, get_tracks, - add_all_tracks, - remove_tracks, remove_section, ) @@ -223,6 +223,7 @@ def add_filters( def provide_track_repository( self, track_files: list[Path], dataset_type: str ) -> tuple[TrackRepository, DetectionMetadata]: + parser: TrackParser if dataset_type == PYTHON: repository = TrackRepository(self.provide_python_track_dataset()) parser = OttrkParser(self.provide_python_detection_parser(repository)) @@ -231,7 +232,10 @@ def provide_track_repository( parser = OttrkParser(self.provide_pandas_detection_parser()) elif dataset_type == PURE_PANDAS: repository = TrackRepository(self.provide_pure_pandas_track_dataset()) - parser = OttrkParser(self.provide_pure_pandas_detection_parser()) + parser = FeathersParser() + elif dataset_type == POLARS: + repository = TrackRepository(self.provide_polars_track_dataset()) + parser = FeathersParser(PolarsTrackGeometryDataset.from_track_dataset) else: raise ValueError(f"Unknown dataset type {dataset_type}") detection_metadata = _fill_track_repository(parser, repository, track_files) @@ -270,7 +274,7 @@ def provide_python_detection_parser( def provide_pure_pandas_track_dataset(self) -> TrackDataset: return FilterByClassPandasTrackDataset( PandasTrackDataset.from_list( - [], PandasTrackGeometryDataset.from_track_dataset + [], PolarsTrackGeometryDataset.from_track_dataset ), self._include_classes, self._exclude_classes, @@ -278,7 +282,12 @@ def provide_pure_pandas_track_dataset(self) -> TrackDataset: def provide_pure_pandas_detection_parser(self) -> PandasDetectionParser: return PandasDetectionParser( - PandasByMaxConfidence(), PandasTrackGeometryDataset.from_track_dataset + PandasByMaxConfidence(), PolarsTrackGeometryDataset.from_track_dataset + ) + + def provide_polars_track_dataset(self) -> TrackDataset: + return PolarsTrackDataset.from_list( + [], PolarsTrackGeometryDataset.from_track_dataset ) def counting_specification(self, save_dir: Path) -> CountingSpecificationDto: @@ -319,7 +328,7 @@ def get_track_parser(self, dataset_type: str = CURRENT_DATASET_TYPE) -> TrackPar elif dataset_type == PANDAS: return OttrkParser(self.provide_pandas_detection_parser()) elif dataset_type == PURE_PANDAS: - return OttrkParser(self.provide_pure_pandas_detection_parser()) + return FeathersParser() else: raise ValueError(f"Unknown dataset type {dataset_type}") @@ -412,6 +421,11 @@ def pandas_track_parser() -> TrackParser: return OttrkParser(detection_parser) +@pytest.fixture +def feathers_track_parser() -> TrackParser: + return FeathersParser() + + @pytest.fixture def use_case_provider_15min( otflow_file: Path, track_file_15min: Path, test_data_tmp_dir: Path @@ -481,11 +495,11 @@ class TestBenchmarkTrackParser: def test_load_15min( self, benchmark: BenchmarkFixture, - pandas_track_parser: OttrkParser, + feathers_track_parser: TrackParser, track_file_15min: Path, ) -> None: benchmark.pedantic( - pandas_track_parser.parse, + feathers_track_parser.parse, args=(track_file_15min,), rounds=self.ROUNDS, iterations=self.ITERATIONS, From 2fe31466861a2c1e5da6e509ac9b4d263365db54 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 16:25:31 +0200 Subject: [PATCH 047/128] Fix track id datatypes --- OTAnalytics/plugin_prototypes/track_visualization/track_viz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index f3eb4d1ba..a25a43ad7 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -48,6 +48,7 @@ W, X, Y, + unpack, ) from OTAnalytics.domain.track_repository import ( TrackListObserver, @@ -207,7 +208,7 @@ def get_data(self) -> DataFrame: "must be index of DataFrame for filtering to work." ) - ids = [track_id.id for track_id in self._filter.get_ids()] + ids = [unpack(track_id) for track_id in self._filter.get_ids()] intersection_of_ids = data.index.unique(level=track.TRACK_ID).intersection(ids) return data.loc[intersection_of_ids] From a2e5e17371eb64f2ca32357430b606aa04e039d6 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 16:54:38 +0200 Subject: [PATCH 048/128] Set tracks after initial cutting --- .../plugin_datastore/python_track_store.py | 2 ++ .../simple/cut_tracks_with_sections.py | 12 +++++++++++- OTAnalytics/plugin_ui/base_application.py | 2 ++ .../simple/test_cut_tracks_with_sections.py | 18 ++++++++++++++---- tests/OTAnalytics/plugin_ui/test_cli.py | 4 ++++ tests/benchmark_otanalytics.py | 6 ++++++ 6 files changed, 39 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 95e735fbf..97d6b864c 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -539,6 +539,8 @@ def as_list(self) -> list[Track]: def split(self, batches: int) -> Sequence["PythonTrackDataset"]: dataset_size = len(self._tracks) + if dataset_size == 0: + return [] batch_size = ceil(dataset_size / batches) dataset_batches = [] diff --git a/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py b/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py index cf8988328..d85b1ebf7 100644 --- a/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py +++ b/OTAnalytics/plugin_intersect/simple/cut_tracks_with_sections.py @@ -8,7 +8,11 @@ GetSectionsById, RemoveSection, ) -from OTAnalytics.application.use_cases.track_repository import GetAllTracks +from OTAnalytics.application.use_cases.track_repository import ( + AddAllTracks, + ClearAllTracks, + GetAllTracks, +) from OTAnalytics.domain.observer import OBSERVER, Subject from OTAnalytics.domain.section import ( Section, @@ -32,12 +36,16 @@ def __init__( self, get_sections_by_id: GetSectionsById, get_tracks: GetAllTracks, + clear_all_tracks: ClearAllTracks, + add_all_tracks: AddAllTracks, remove_section: RemoveSection, ) -> None: self._subject: Subject[CutTracksDto] = Subject[CutTracksDto]() self._get_sections_by_id = get_sections_by_id self._get_tracks = get_tracks + self._clear_all_tracks = clear_all_tracks + self._add_all_tracks = add_all_tracks self._remove_section = remove_section def __call__( @@ -47,6 +55,8 @@ def __call__( cut_tracks_dataset, ids_of_cut_tracks = track_dataset.cut_with_section( cutting_section, cutting_section.get_offset(EventType.SECTION_ENTER) ) + self._clear_all_tracks() + self._add_all_tracks(cut_tracks_dataset) if not preserve_cutting_section: self._remove_section(cutting_section.id) self._subject.notify( diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index b13ef9aa7..1e21c080c 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -798,6 +798,8 @@ def cut_tracks_intersecting_section(self) -> CutTracksIntersectingSection: return SimpleCutTracksIntersectingSection( self.get_sections_by_id, self.get_all_tracks, + self.clear_all_tracks, + self.add_all_tracks, self.remove_section, ) diff --git a/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py b/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py index e2dc36e03..137380185 100644 --- a/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py +++ b/tests/OTAnalytics/plugin_intersect/simple/test_cut_tracks_with_sections.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, patch +from unittest.mock import Mock, call, patch import pytest @@ -7,6 +7,8 @@ RemoveSection, ) from OTAnalytics.application.use_cases.track_repository import ( + AddAllTracks, + ClearAllTracks, GetTracksWithoutSingleDetections, ) from OTAnalytics.domain.section import ( @@ -52,6 +54,8 @@ def test_cut(self, cutting_section: LineSection) -> None: get_sections_by_id = Mock(spec=GetSectionsById) get_tracks = Mock(spec=GetTracksWithoutSingleDetections, return_value=[track]) + clear_all_tracks = Mock(spec=ClearAllTracks) + add_all_tracks = Mock(spec=AddAllTracks) cut_tracks_dataset = Mock() track_dataset = Mock() track_dataset.cut_with_section.return_value = (cut_tracks_dataset, {track_id}) @@ -62,13 +66,19 @@ def test_cut(self, cutting_section: LineSection) -> None: cut_tracks_intersecting_section = SimpleCutTracksIntersectingSection( get_sections_by_id, get_tracks, + clear_all_tracks, + add_all_tracks, remove_section, ) cut_tracks_intersecting_section(cutting_section, False) cut_tracks_intersecting_section(cutting_section, True) assert get_tracks.as_dataset.call_count == 2 - + assert clear_all_tracks.call_count == 2 + assert add_all_tracks.call_args_list == [ + call(cut_tracks_dataset), + call(cut_tracks_dataset), + ] remove_section.assert_called_once_with(cutting_section.id) def test_notify_sections( @@ -83,7 +93,7 @@ def test_notify_sections( with patch.object(SimpleCutTracksIntersectingSection, "__call__") as call_mock: cut_tracks_intersecting_section = SimpleCutTracksIntersectingSection( - get_sections_by_id, Mock(), Mock() + get_sections_by_id, Mock(), Mock(), Mock(), Mock() ) cut_tracks_intersecting_section.notify_sections( SectionRepositoryEvent.create_added(section_ids) @@ -98,7 +108,7 @@ def test_notify_sections( def test_register(self, mock_subject_register: Mock) -> None: observer = Mock() cut_tracks_intersecting_section = SimpleCutTracksIntersectingSection( - Mock(), Mock(), Mock() + Mock(), Mock(), Mock(), Mock(), Mock() ) cut_tracks_intersecting_section.register(observer) mock_subject_register.assert_called_once_with(observer) diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index 3d68a3809..ca34fb306 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -416,6 +416,8 @@ def cli_dependencies_old(self) -> dict[str, Any]: cut_tracks = SimpleCutTracksIntersectingSection( GetSectionsById(section_repository), get_all_tracks, + clear_all_tracks, + add_all_tracks, RemoveSection(section_repository), ) apply_cli_cuts = ApplyCliCuts(cut_tracks, TrackRepositorySize(track_repository)) @@ -509,6 +511,8 @@ def cli_dependencies(self) -> dict[str, Any]: cut_tracks = SimpleCutTracksIntersectingSection( GetSectionsById(section_repository), get_all_tracks, + clear_all_tracks, + add_all_tracks, RemoveSection(section_repository), ) apply_cli_cuts = ApplyCliCuts(cut_tracks, TrackRepositorySize(track_repository)) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index a093fbd66..5cf5fdc93 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -29,6 +29,8 @@ RemoveSection, ) from OTAnalytics.application.use_cases.track_repository import ( + AddAllTracks, + ClearAllTracks, GetAllTracks, GetTracksWithoutSingleDetections, ) @@ -205,10 +207,14 @@ def get_export_counts(self) -> ExportCounts: def get_cut_tracks(self) -> CutTracksIntersectingSection: get_sections_by_id = GetSectionsById(self._section_repository) get_tracks = GetAllTracks(self._track_repository) + clear_all_tracks = ClearAllTracks(self._track_repository) + add_all_tracks = AddAllTracks(self._track_repository) remove_section = RemoveSection(self._section_repository) return SimpleCutTracksIntersectingSection( get_sections_by_id, get_tracks, + clear_all_tracks, + add_all_tracks, remove_section, ) From 9a418a8d205a4450fb6967174d20818361fab782 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 20 Aug 2025 17:43:47 +0200 Subject: [PATCH 049/128] Fix track ids after cutting --- .../plugin_datastore/polars_track_store.py | 4 +--- .../polars_geometry_store.py | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 44ab394fa..cb715184b 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -623,9 +623,7 @@ def cut_with_section( old_track_ids = self._dataset.rename({track.TRACK_ID: "old_track_id"}) result = old_track_ids.join(new_track_ids, on=ROW_ID, how="left") - result = result.with_columns( - (pl.col(track.TRACK_ID).backward_fill().over("old_track_id")) - ).drop("old_track_id") + result = result.drop("old_track_id") return ( PolarsTrackDataset.from_dataframe(result, self.track_geometry_factory), set(), diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 69de20570..410844e08 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -351,6 +351,7 @@ def find_line_intersections( ] ).select( [ + ROW_ID, track.TRACK_ID, START_X, START_Y, @@ -962,7 +963,7 @@ def track_ids_after_cut(self, section: Section) -> pl.DataFrame: coordinates = section.get_coordinates() offset = get_section_offset(section) - if section.get_type() == SectionType.CUTTING: + if section.get_type() in [SectionType.CUTTING, SectionType.LINE]: # For line sections, check if any track segment intersects with any leg # of the line. A leg is formed by each consecutive pair of coordinates. for i in range(len(coordinates) - 1): @@ -995,7 +996,24 @@ def track_ids_after_cut(self, section: Section) -> pl.DataFrame: .select([ROW_ID, TRACK_ID, INTERSECTS]) ) - results = result.with_columns(pl.col(INTERSECTS).cum_sum().alias("cum_sum")) + COLUMN_ORDER = [ROW_ID, TRACK_ID, INTERSECTS, "cum_sum", "order"] + results = ( + result.with_columns( + pl.col(INTERSECTS).cum_sum().over(TRACK_ID).alias("cum_sum") + ) + .with_columns(pl.lit(1).alias("order")) + .select(COLUMN_ORDER) + ) + temp = ( + results.group_by(TRACK_ID) + .first() + .with_columns(pl.lit(0).alias("order")) + .with_columns(pl.lit(0, pl.UInt32).alias("cum_sum")) + .with_columns(pl.col(ROW_ID) - 1) + .select(COLUMN_ORDER) + ) + temp_r = pl.concat([results, temp]) + results = temp_r.sort(by=[TRACK_ID, ROW_ID, "order"]) return results.with_columns( pl.col(TRACK_ID) + "_" + pl.col("cum_sum").cast(pl.Utf8).alias(TRACK_ID) ).select([ROW_ID, TRACK_ID]) From 5bc8a9ac287dca6a8e330faafc86ac73cd0063a4 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 27 Aug 2025 14:28:10 +0200 Subject: [PATCH 050/128] Add TrackIdSet to efficiently handle TrackIds --- .../track_dataset/filtered_track_dataset.py | 3 +- .../domain/track_dataset/track_dataset.py | 20 ++++++- .../plugin_datastore/polars_track_store.py | 58 +++++++++++++++++-- .../plugin_datastore/python_track_store.py | 35 ++++++++++- OTAnalytics/plugin_datastore/track_store.py | 17 +++--- .../test_python_track_store.py | 8 +-- .../plugin_datastore/test_track_store.py | 8 +-- tests/utils/assertions.py | 3 +- 8 files changed, 125 insertions(+), 27 deletions(-) diff --git a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py index 695046836..9f60784a9 100644 --- a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py +++ b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py @@ -8,6 +8,7 @@ from OTAnalytics.domain.track_dataset.track_dataset import ( IntersectionPoint, TrackDataset, + TrackIdSet, TrackSegmentDataset, ) @@ -18,7 +19,7 @@ def _filter(self) -> TrackDataset: raise NotImplementedError @property - def track_ids(self) -> frozenset[TrackId]: + def track_ids(self) -> TrackIdSet: return self._filter().track_ids @property diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 58e0d20a1..fb0bc6bd0 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -64,10 +64,28 @@ def apply(self, consumer: Callable[[dict], None]) -> None: raise NotImplementedError +class TrackIdSet(ABC): + @abstractmethod + def __iter__(self) -> Iterator[TrackId]: + raise NotImplementedError + + @abstractmethod + def __len__(self) -> int: + raise NotImplementedError + + @abstractmethod + def intersection(self, other: "TrackIdSet") -> "TrackIdSet": + raise NotImplementedError + + @abstractmethod + def union(self, other: "TrackIdSet") -> "TrackIdSet": + raise NotImplementedError + + class TrackDataset(ABC): @property @abstractmethod - def track_ids(self) -> frozenset[TrackId]: + def track_ids(self) -> TrackIdSet: raise NotImplementedError @property diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index cb715184b..a9a34b8f8 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -40,6 +40,7 @@ IntersectionPoint, TrackDataset, TrackDoesNotExistError, + TrackIdSet, TrackSegmentDataset, ) from OTAnalytics.domain.types import EventType @@ -273,12 +274,11 @@ def drop_row_id(df: pl.DataFrame) -> pl.DataFrame: class PolarsTrackDataset(TrackDataset, PolarsDataFrameProvider): @property - def track_ids(self) -> frozenset[TrackId]: + def track_ids(self) -> TrackIdSet: if self._dataset.is_empty(): - return frozenset() - return frozenset( - TrackId(_id) for _id in self._dataset.get_column(track.TRACK_ID).unique() - ) + return PolarsTrackIdSet() + unique_track_ids = self._dataset.get_column(track.TRACK_ID).unique() + return PolarsTrackIdSet(unique_track_ids) @property def first_occurrence(self) -> datetime | None: @@ -874,3 +874,51 @@ def area_section_to_shapely(section: Section) -> BaseGeometry: geometry = shapely.Polygon([(c.x, c.y) for c in section.get_coordinates()]) prepare(geometry) return geometry + + +class PolarsTrackIdSet(TrackIdSet): + """ + Polars-based implementation of TrackIdSet using a Series for optimal performance. + """ + + def __init__(self, track_ids: Iterable[TrackId] | pl.Series | None = None): + if track_ids is None: + self._series = pl.Series([], dtype=pl.String) + elif isinstance(track_ids, pl.Series): + self._series = track_ids.unique().sort() + else: + # Convert TrackId objects to strings + id_strings = [unpack(track_id) for track_id in track_ids] + self._series = pl.Series(id_strings, dtype=pl.String).unique().sort() + + def __iter__(self) -> Iterator[TrackId]: + for id_str in self._series: + yield TrackId(id_str) + + def __len__(self) -> int: + return len(self._series) + + def intersection(self, other: "TrackIdSet") -> "TrackIdSet": + if isinstance(other, PolarsTrackIdSet): + # Efficient intersection using Polars operations + intersected_series = self._series.filter(self._series.is_in(other._series)) + return PolarsTrackIdSet(intersected_series) + else: + # Convert other to set of strings for intersection + other_strings = {unpack(track_id) for track_id in other} + intersected_series = self._series.filter( + self._series.is_in(list(other_strings)) + ) + return PolarsTrackIdSet(intersected_series) + + def union(self, other: "TrackIdSet") -> "TrackIdSet": + if isinstance(other, PolarsTrackIdSet): + # Efficient union using Polars operations + combined_series = pl.concat([self._series, other._series]).unique().sort() + return PolarsTrackIdSet(combined_series) + else: + # Convert other to strings and combine + other_strings = [unpack(track_id) for track_id in other] + other_series = pl.Series(other_strings, dtype=pl.String) + combined_series = pl.concat([self._series, other_series]).unique().sort() + return PolarsTrackIdSet(combined_series) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 97d6b864c..e4de7ca97 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime from math import ceil -from typing import Callable, Iterable, Optional, Sequence +from typing import Callable, Iterable, Iterator, Optional, Sequence from more_itertools import batched @@ -40,6 +40,7 @@ TrackDataset, TrackDoesNotExistError, TrackGeometryDataset, + TrackIdSet, TrackSegmentDataset, ) from OTAnalytics.plugin_intersect.shapely.mapping import ShapelyMapper @@ -384,8 +385,8 @@ class PythonTrackDataset(TrackDataset): """Pure Python implementation of a TrackDataset.""" @property - def track_ids(self) -> frozenset[TrackId]: - return frozenset(self._tracks.keys()) + def track_ids(self) -> TrackIdSet: + return PythonTrackIdSet(self._tracks.keys()) @property def first_occurrence(self) -> datetime | None: @@ -938,3 +939,31 @@ def _build_detections(self) -> list[Detection]: ) ) return new_detections + + +class PythonTrackIdSet(TrackIdSet): + """Pure Python implementation of TrackIdSet using a set for storage.""" + + def __init__(self, track_ids: Iterable[TrackId] | None = None): + if track_ids is None: + self._ids: set[TrackId] = set() + else: + self._ids = set(track_ids) + + def __iter__(self) -> Iterator[TrackId]: + return iter(self._ids) + + def __len__(self) -> int: + return len(self._ids) + + def intersection(self, other: "TrackIdSet") -> "TrackIdSet": + if isinstance(other, PythonTrackIdSet): + return PythonTrackIdSet(self._ids & other._ids) + else: + return PythonTrackIdSet(self._ids & set(other)) + + def union(self, other: "TrackIdSet") -> "TrackIdSet": + if isinstance(other, PythonTrackIdSet): + return PythonTrackIdSet(self._ids | other._ids) + else: + return PythonTrackIdSet(self._ids | set(other)) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index b41dbdff4..ea929624e 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -36,8 +36,10 @@ TrackDataset, TrackDoesNotExistError, TrackGeometryDataset, + TrackIdSet, TrackSegmentDataset, ) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_parser import ottrk_dataformat RANK = "rank" @@ -250,15 +252,14 @@ def get_data(self) -> DataFrame: class PandasTrackDataset(TrackDataset, PandasDataFrameProvider): @property - def track_ids(self) -> frozenset[TrackId]: + def track_ids(self) -> TrackIdSet: if self._dataset.empty: - return frozenset() - return frozenset( - [ - TrackId(_id) - for _id in self._dataset.index.get_level_values(LEVEL_TRACK_ID) - ] - ) + return PythonTrackIdSet() + track_ids = [ + TrackId(_id) + for _id in self._dataset.index.get_level_values(LEVEL_TRACK_ID).unique() + ] + return PythonTrackIdSet(track_ids) @property def first_occurrence(self) -> datetime | None: diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_store.py b/tests/OTAnalytics/plugin_datastore/test_python_track_store.py index cb1163df3..7ac4f2da3 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_store.py @@ -668,11 +668,11 @@ def test_track_ids( pedestrian_track: Track, ) -> None: dataset = PythonTrackDataset(ShapelyTrackGeometryDataset.from_track_dataset) - assert dataset.track_ids == frozenset() + assert len(dataset.track_ids) == 0 + assert set(dataset.track_ids) == set() updated_dataset = dataset.add_all([car_track, pedestrian_track]) - assert updated_dataset.track_ids == frozenset( - [car_track.id, pedestrian_track.id] - ) + assert len(updated_dataset.track_ids) == 2 + assert set(updated_dataset.track_ids) == {car_track.id, pedestrian_track.id} def test_empty(self, car_track: Track) -> None: empty_dataset = PythonTrackDataset( diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index b3f42f53e..8ba380070 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -607,11 +607,11 @@ def test_track_ids( pedestrian_track: Track, ) -> None: dataset = PandasTrackDataset(track_geometry_factory) - assert dataset.track_ids == frozenset() + assert len(dataset.track_ids) == 0 + assert set(dataset.track_ids) == set() updated_dataset = dataset.add_all([car_track, pedestrian_track]) - assert updated_dataset.track_ids == frozenset( - [car_track.id, pedestrian_track.id] - ) + assert len(updated_dataset.track_ids) == 2 + assert set(updated_dataset.track_ids) == {car_track.id, pedestrian_track.id} def test_empty( self, track_geometry_factory: TRACK_GEOMETRY_FACTORY, car_track: Track diff --git a/tests/utils/assertions.py b/tests/utils/assertions.py index 3fd90ac94..a37ecc609 100644 --- a/tests/utils/assertions.py +++ b/tests/utils/assertions.py @@ -33,7 +33,8 @@ def assert_equal_track_properties(actual: Track, expected: Track) -> None: def assert_track_datasets_equal(actual: TrackDataset, expected: TrackDataset) -> None: - assert actual.track_ids == expected.track_ids + assert len(actual.track_ids) == len(expected.track_ids) + assert set(actual.track_ids) == set(expected.track_ids) for actual_track in actual.as_list(): if expected_track := expected.get_for(actual_track.id): From f30e092521a35252488dc275630a3a787e897538 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 29 Aug 2025 20:22:21 +0200 Subject: [PATCH 051/128] WORK IN PROGRESS: Use TrackIdSet where possible --- .../application/analysis/traffic_counting.py | 5 +- .../use_cases/highlight_intersections.py | 26 ++++--- .../use_cases/inside_cutting_section.py | 21 ++--- .../application/use_cases/track_repository.py | 15 ++-- .../application/use_cases/track_statistics.py | 2 +- OTAnalytics/domain/track.py | 14 ---- .../domain/track_dataset/track_dataset.py | 49 +++++++----- OTAnalytics/domain/track_id_provider.py | 16 ++++ OTAnalytics/domain/track_repository.py | 8 +- OTAnalytics/plugin_cli/cli_application.py | 2 +- .../plugin_datastore/polars_track_id_set.py | 65 ++++++++++++++++ .../plugin_datastore/polars_track_store.py | 76 ++++--------------- .../plugin_datastore/python_track_store.py | 31 +++++++- .../polars_geometry_store.py | 6 +- OTAnalytics/plugin_datastore/track_store.py | 19 ++++- OTAnalytics/plugin_filter/pandas_track_id.py | 8 +- .../svz/number_of_tracks_to_be_validated.py | 8 +- .../track_visualization/track_viz.py | 2 +- OTAnalytics/plugin_ui/base_application.py | 2 +- OTAnalytics/plugin_ui/gui_application.py | 2 +- .../plugin_ui/visualization/visualization.py | 2 +- .../use_cases/test_highlight_intersections.py | 3 +- .../use_cases/test_inside_cutting_section.py | 26 ++++--- .../use_cases/test_track_repository.py | 6 +- .../use_cases/test_track_statistics.py | 3 +- .../track_visualization/test_track_viz.py | 2 +- 26 files changed, 255 insertions(+), 164 deletions(-) create mode 100644 OTAnalytics/domain/track_id_provider.py create mode 100644 OTAnalytics/plugin_datastore/polars_track_id_set.py diff --git a/OTAnalytics/application/analysis/traffic_counting.py b/OTAnalytics/application/analysis/traffic_counting.py index 51680f232..f382d40e5 100644 --- a/OTAnalytics/application/analysis/traffic_counting.py +++ b/OTAnalytics/application/analysis/traffic_counting.py @@ -18,6 +18,7 @@ from OTAnalytics.domain.flow import Flow, FlowRepository from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet LEVEL_FROM_SECTION = "from section" LEVEL_TO_SECTION = "to section" @@ -597,7 +598,9 @@ def road_user_ids(self) -> list[str]: Returns: list[str]: the road user ids. """ - return sorted([assignment.road_user for assignment in self._assignments]) + return PythonTrackIdSet( + sorted([assignment.road_user for assignment in self._assignments]) + ) def __init__(self, assignments: list[RoadUserAssignment]) -> None: self._assignments = assignments.copy() diff --git a/OTAnalytics/application/use_cases/highlight_intersections.py b/OTAnalytics/application/use_cases/highlight_intersections.py index 3252e9cff..453cf0ad5 100644 --- a/OTAnalytics/application/use_cases/highlight_intersections.py +++ b/OTAnalytics/application/use_cases/highlight_intersections.py @@ -13,7 +13,9 @@ from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.flow import FlowId, FlowRepository from OTAnalytics.domain.section import SectionId -from OTAnalytics.domain.track import Track, TrackId, TrackIdProvider +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository @@ -53,7 +55,7 @@ def __init__( self._get_section_by_id = get_section_by_id self._intersection_repository = intersection_repository - def get_ids(self) -> set[TrackId]: + def get_ids(self) -> TrackIdSet: currently_selected_sections = self._section_state.selected_sections.get() return TracksIntersectingGivenSections( set(currently_selected_sections), @@ -87,7 +89,7 @@ def __init__( self._get_section_by_id = get_sections_by_id self._intersection_repository = intersection_repository - def get_ids(self) -> set[TrackId]: + def get_ids(self) -> TrackIdSet: ids_non_cutting_sections = { section.id for section in self._get_all_sections() @@ -123,7 +125,7 @@ def __init__( self._get_section_by_id = get_section_by_id self._intersection_repository = intersection_repository - def get_ids(self) -> set[TrackId]: + def get_ids(self) -> TrackIdSet: ids_all_sections = {section.id for section in self._get_all_sections()} return TracksIntersectingGivenSections( ids_all_sections, @@ -155,15 +157,15 @@ def __init__( self._get_section_by_id = get_section_by_id self._intersection_repository = intersection_repository - def get_ids(self) -> set[TrackId]: + def get_ids(self) -> TrackIdSet: existing_intersections = self._intersection_repository.get(self._section_ids) section_ids_to_process = self._section_ids - existing_intersections.keys() new_intersections = self._calculate_new_intersections(section_ids_to_process) - return set.union(new_intersections, *existing_intersections.values()) + return new_intersections.union(*existing_intersections.values()) def _calculate_new_intersections( self, section_ids_to_process: set[SectionId] - ) -> set[TrackId]: + ) -> TrackIdSet: new_intersections: set[TrackId] = set() if section_ids_to_process: sections = self._get_section_by_id(section_ids_to_process) @@ -189,7 +191,7 @@ def __init__( self._track_id_provider = track_id_provider self._track_repository = track_repository - def get_ids(self) -> Iterable[TrackId]: + def get_ids(self) -> TrackIdSet: all_track_ids = {track.id for track in self._track_repository.get_all()} assigned_tracks = set(self._track_id_provider.get_ids()) return all_track_ids - assigned_tracks @@ -217,7 +219,7 @@ def __init__( self._flow_repository = flow_repository self._flow_state = flow_state - def get_ids(self) -> Iterable[TrackId]: + def get_ids(self) -> TrackIdSet: events = self._event_repository.get_all() # All flows must be passed to assigner to ensure that a track potentially # belonging to several flows is assigned to the correct one. @@ -251,7 +253,7 @@ def __init__( self._event_repository = event_repository self._flow_repository = flow_repository - def get_ids(self) -> Iterable[TrackId]: + def get_ids(self) -> TrackIdSet: all_flow_ids = [flow.id for flow in self._flow_repository.get_all()] return TracksAssignedToGivenFlows( self._assigner, self._event_repository, self._flow_repository, all_flow_ids @@ -280,7 +282,7 @@ def __init__( self._flow_repository = flow_repository self._flows = list(flow_ids) - def get_ids(self) -> Iterable[TrackId]: + def get_ids(self) -> TrackIdSet: events = self._event_repository.get_all() # All flows must be passed to assigner to ensure that a track potentially # belonging to several flows is assigned to the correct one. @@ -314,7 +316,7 @@ def __init__( self._track_repository = track_repository self._track_view_state = track_view_state - def get_ids(self) -> Iterable[TrackId]: + def get_ids(self) -> TrackIdSet: if self._other: tracks = self._get_other_track_ids() else: diff --git a/OTAnalytics/application/use_cases/inside_cutting_section.py b/OTAnalytics/application/use_cases/inside_cutting_section.py index c48bf0d5e..48735f560 100644 --- a/OTAnalytics/application/use_cases/inside_cutting_section.py +++ b/OTAnalytics/application/use_cases/inside_cutting_section.py @@ -3,8 +3,11 @@ from OTAnalytics.application.use_cases.section_repository import GetCuttingSections from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.section import SectionListObserver, SectionRepositoryEvent -from OTAnalytics.domain.track import TrackId, TrackIdProvider +from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet, EmptyTrackIdSet +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackListObserver, TrackRepositoryEvent +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class TrackIdsInsideCuttingSections(TrackIdProvider): @@ -14,11 +17,11 @@ def __init__( self._get_tracks = get_tracks self._get_cutting_sections = get_cutting_sections - def get_ids(self) -> set[TrackId]: + def get_ids(self) -> TrackIdSet: track_dataset = self._get_tracks.as_dataset() cutting_sections = self._get_cutting_sections() if not cutting_sections: - return set() + return PythonTrackIdSet() return track_dataset.ids_inside(cutting_sections) @@ -32,9 +35,9 @@ def __init__( get_cutting_sections: GetCuttingSections, ) -> None: super().__init__(get_tracks, get_cutting_sections) - self._cached_ids: set[TrackId] = self.__get_empty_cache() + self._cached_ids: TrackIdSet = self.__get_empty_cache() - def get_ids(self) -> set[TrackId]: + def get_ids(self) -> TrackIdSet: if self._cached_ids == self.__get_empty_cache(): self._cached_ids = super().get_ids() @@ -53,8 +56,8 @@ def notify_sections(self, sections: SectionRepositoryEvent) -> None: def _reset_cache(self) -> None: self._cached_ids = self.__get_empty_cache() - def _remove_track_ids(self, remove_track_ids: Iterable[TrackId]) -> None: - self._cached_ids = self._cached_ids.difference(set(remove_track_ids)) + def _remove_track_ids(self, remove_track_ids: TrackIdSet) -> None: + self._cached_ids = self._cached_ids.difference(remove_track_ids) - def __get_empty_cache(self) -> set[TrackId]: - return set() + def __get_empty_cache(self) -> TrackIdSet: + return EmptyTrackIdSet() diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index 4979c1700..de58269e1 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -2,8 +2,9 @@ from typing import Iterable from OTAnalytics.application.logger import logger -from OTAnalytics.domain.track import Track, TrackId, TrackIdProvider, pack, unpack -from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset, TrackIdSet +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import ( RemoveMultipleTracksError, TrackFileRepository, @@ -15,7 +16,7 @@ class AllTrackIdsProvider(TrackIdProvider): def __init__(self, track_repository: TrackRepository) -> None: self._track_repository = track_repository - def get_ids(self) -> Iterable[TrackId]: + def get_ids(self) -> TrackIdSet: return self._track_repository.get_all_ids() @@ -24,10 +25,10 @@ def __init__(self, other: TrackIdProvider, by: TrackIdProvider): self._other = other self._by = by - def get_ids(self) -> set[TrackId]: - other_ids = set({unpack(id) for id in self._other.get_ids()}) - self_ids = set({unpack(id) for id in self._by.get_ids()}) - return {pack(id) for id in other_ids.intersection(self_ids)} + def get_ids(self) -> TrackIdSet: + other_ids = self._other.get_ids() + self_ids = self._by.get_ids() + return other_ids.intersection(self_ids) class GetAllTracks: diff --git a/OTAnalytics/application/use_cases/track_statistics.py b/OTAnalytics/application/use_cases/track_statistics.py index 1fa38db17..befd87f75 100644 --- a/OTAnalytics/application/use_cases/track_statistics.py +++ b/OTAnalytics/application/use_cases/track_statistics.py @@ -7,7 +7,7 @@ from OTAnalytics.application.use_cases.number_of_tracks_to_be_validated import ( NumberOfTracksToBeValidated, ) -from OTAnalytics.domain.track import TrackIdProvider +from OTAnalytics.domain.track_id_provider import TrackIdProvider @dataclass diff --git a/OTAnalytics/domain/track.py b/OTAnalytics/domain/track.py index c5314c779..150acbd43 100644 --- a/OTAnalytics/domain/track.py +++ b/OTAnalytics/domain/track.py @@ -1,7 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime -from typing import Iterable from PIL import Image @@ -301,19 +300,6 @@ def calculate(self, detections: list[Detection]) -> str: raise NotImplementedError -class TrackIdProvider(ABC): - """Interface to provide track ids.""" - - @abstractmethod - def get_ids(self) -> Iterable[TrackId]: - """Provide track ids. - - Returns: - Iterable[TrackId]: the track ids. - """ - pass - - class TrackBuilderError(Exception): pass diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index fb0bc6bd0..c5ba537dd 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -65,6 +65,11 @@ def apply(self, consumer: Callable[[dict], None]) -> None: class TrackIdSet(ABC): + + @abstractmethod + def __eq__(self, other: Any) -> bool: + raise NotImplementedError + @abstractmethod def __iter__(self) -> Iterator[TrackId]: raise NotImplementedError @@ -81,6 +86,24 @@ def intersection(self, other: "TrackIdSet") -> "TrackIdSet": def union(self, other: "TrackIdSet") -> "TrackIdSet": raise NotImplementedError + @abstractmethod + def difference(self, other: "TrackIdSet") -> "TrackIdSet": + raise NotImplementedError + + +class EmptyTrackIdSet(TrackIdSet): + def __eq__(self, other: Any) -> bool: + return isinstance(other, EmptyTrackIdSet) + def __iter__(self) -> Iterator[TrackId]: + return iter([]) + def __len__(self) -> int: + return 0 + def intersection(self, other: "TrackIdSet") -> "TrackIdSet": + return other + def union(self, other: "TrackIdSet") -> "TrackIdSet": + return self + def difference(self, other: "TrackIdSet") -> "TrackIdSet": + return self class TrackDataset(ABC): @property @@ -155,7 +178,7 @@ def as_list(self) -> list[Track]: @abstractmethod def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: + ) -> TrackIdSet: """Return a set of tracks intersecting a set of sections. Args: @@ -235,7 +258,7 @@ def get_last_segments(self) -> TrackSegmentDataset: @abstractmethod def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple["TrackDataset", set[TrackId]]: + ) -> tuple["TrackDataset", TrackId]: """Use section to cut track with TrackDataset. Args: @@ -249,7 +272,7 @@ def cut_with_section( raise NotImplementedError @abstractmethod - def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: + def get_max_confidences_for(self, track_ids: TrackIdSet) -> dict[str, float]: """Get max confidences for given track ids. Args: @@ -264,7 +287,7 @@ def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: raise NotImplementedError @abstractmethod - def revert_cuts_for(self, original_track_ids: frozenset[TrackId]) -> "TrackDataset": + def revert_cuts_for(self, original_track_ids: TrackIdSet) -> "TrackDataset": """ Reverses the effects of track cutting operations for the specified original track IDs. @@ -279,22 +302,8 @@ def revert_cuts_for(self, original_track_ids: frozenset[TrackId]) -> "TrackDatas """ raise NotImplementedError - def ids_inside(self, sections: list[Section]) -> set[TrackId]: - results: set[TrackId] = set() - for cutting_section in sections: - offset = cutting_section.get_offset(EventType.SECTION_ENTER) - # set of all tracks where at least one coordinate is contained - # by at least one cutting section - results.update( - set( - track_id - for track_id, section_data in ( - self.contained_by_sections([cutting_section], offset).items() - ) - if contains_true(section_data) - ) - ) - return results + def ids_inside(self, sections: list[Section]) -> TrackIdSet: + raise NotImplementedError class TrackGeometryDataset(ABC): diff --git a/OTAnalytics/domain/track_id_provider.py b/OTAnalytics/domain/track_id_provider.py new file mode 100644 index 000000000..6767b9191 --- /dev/null +++ b/OTAnalytics/domain/track_id_provider.py @@ -0,0 +1,16 @@ +from abc import ABC, abstractmethod + +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet + + +class TrackIdProvider(ABC): + """Interface to provide track ids.""" + + @abstractmethod + def get_ids(self) -> TrackIdSet: + """Provide track ids. + + Returns: + TrackIdSet: the track ids. + """ + pass diff --git a/OTAnalytics/domain/track_repository.py b/OTAnalytics/domain/track_repository.py index 77d97b381..975096c70 100644 --- a/OTAnalytics/domain/track_repository.py +++ b/OTAnalytics/domain/track_repository.py @@ -7,13 +7,13 @@ from OTAnalytics.application.logger import logger from OTAnalytics.domain.observer import OBSERVER, Subject from OTAnalytics.domain.track import Track, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset +from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset, TrackIdSet @dataclass(frozen=True) class TrackRepositoryEvent: - added: frozenset[TrackId] - removed: frozenset[TrackId] + added: TrackIdSet + removed: TrackIdSet @staticmethod def create_added(tracks: Iterable[TrackId]) -> "TrackRepositoryEvent": @@ -166,7 +166,7 @@ def get_all(self) -> TrackDataset: """ return self._dataset - def get_all_ids(self) -> Iterable[TrackId]: + def get_all_ids(self) -> TrackIdSet: """Get all track ids in this repository. Returns: diff --git a/OTAnalytics/plugin_cli/cli_application.py b/OTAnalytics/plugin_cli/cli_application.py index 42b2996c3..7e688f3fe 100644 --- a/OTAnalytics/plugin_cli/cli_application.py +++ b/OTAnalytics/plugin_cli/cli_application.py @@ -7,7 +7,7 @@ ) from OTAnalytics.application.use_cases.track_repository import AllTrackIdsProvider from OTAnalytics.domain.progress import ProgressbarBuilder -from OTAnalytics.domain.track import TrackIdProvider +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.plugin_progress.tqdm_progressbar import TqdmBuilder from OTAnalytics.plugin_prototypes.eventlist_exporter.eventlist_exporter import ( provide_available_eventlist_exporter, diff --git a/OTAnalytics/plugin_datastore/polars_track_id_set.py b/OTAnalytics/plugin_datastore/polars_track_id_set.py new file mode 100644 index 000000000..ecbf0a06f --- /dev/null +++ b/OTAnalytics/plugin_datastore/polars_track_id_set.py @@ -0,0 +1,65 @@ +from typing import Iterable, Iterator, Any + +import polars as pl + +from OTAnalytics.domain.track import TrackId, unpack +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet + + +class PolarsTrackIdSet(TrackIdSet): + """ + Polars-based implementation of TrackIdSet using a Series for optimal performance. + """ + + def __init__(self, track_ids: Iterable[TrackId] | pl.Series | None = None): + if track_ids is None: + self._series = pl.Series([], dtype=pl.String) + elif isinstance(track_ids, pl.Series): + self._series = track_ids.unique().sort() + else: + # Convert TrackId objects to strings + id_strings = [unpack(track_id) for track_id in track_ids] + self._series = pl.Series(id_strings, dtype=pl.String).unique().sort() + + def __eq__(self, other: Any) -> bool: + if isinstance(other, PolarsTrackIdSet): + return self._series.equals(other._series) + elif isinstance(other, TrackIdSet): + return self.__eq__(PolarsTrackIdSet(other)) + return False + + def __iter__(self) -> Iterator[TrackId]: + for id_str in self._series: + yield TrackId(id_str) + + def __len__(self) -> int: + return len(self._series) + + def intersection(self, other: "TrackIdSet") -> "TrackIdSet": + if isinstance(other, PolarsTrackIdSet): + # Efficient intersection using Polars operations + intersected_series = self._series.filter(self._series.is_in(other._series)) + return PolarsTrackIdSet(intersected_series) + else: + # Convert other to set of strings for intersection + other_strings = {unpack(track_id) for track_id in other} + intersected_series = self._series.filter( + self._series.is_in(list(other_strings)) + ) + return PolarsTrackIdSet(intersected_series) + + def union(self, other: "TrackIdSet") -> "TrackIdSet": + if isinstance(other, PolarsTrackIdSet): + # Efficient union using Polars operations + combined_series = pl.concat([self._series, other._series]).unique().sort() + return PolarsTrackIdSet(combined_series) + else: + # Convert other to strings and combine + other_strings = [unpack(track_id) for track_id in other] + other_series = pl.Series(other_strings, dtype=pl.String) + combined_series = pl.concat([self._series, other_series]).unique().sort() + return PolarsTrackIdSet(combined_series) + + def difference(self, other: "TrackIdSet") -> "TrackIdSet": + # TODO implement + raise NotImplementedError \ No newline at end of file diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index a9a34b8f8..0e2532ae2 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -44,6 +44,7 @@ TrackSegmentDataset, ) from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSet from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( ROW_ID, PolarsTrackGeometryDataset, @@ -447,7 +448,7 @@ def __len__(self) -> int: def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: + ) -> TrackIdSet: geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.intersecting_tracks(sections) @@ -478,8 +479,8 @@ def intersection_points( geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.intersection_points(sections) - def ids_inside(self, sections: list[Section]) -> set[TrackId]: - result: set[TrackId] = set() + def ids_inside(self, sections: list[Section]) -> TrackIdSet: + result: TrackIdSet = PolarsTrackIdSet() for _section in sections: section_geom = area_section_to_shapely(_section) offset = _section.get_offset(EventType.SECTION_ENTER) @@ -491,12 +492,13 @@ def ids_inside(self, sections: list[Section]) -> set[TrackId]: ).to_numpy() contains_masks = contains(section_geom, x_coords, y_coords) - result.update( - self._dataset.filter(contains_masks) - .select(pl.col(track.TRACK_ID)) - .unique() - .to_series() - .to_list() + result = result.union( + PolarsTrackIdSet( + self._dataset.filter(contains_masks) + .select(pl.col(track.TRACK_ID)) + .unique() + .to_series() + ) ) return result @@ -669,7 +671,7 @@ def filter_by_min_detection_length(self, length: int) -> TrackDataset: self.track_geometry_factory, filtered_dataset, calculator=self.calculator ) - def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: + def get_max_confidences_for(self, track_ids: TrackIdSet) -> dict[str, float]: if not track_ids or self._dataset.is_empty(): return {} @@ -696,9 +698,7 @@ def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: "Some tracks do not exist in dataset with given id" ) from cause - def revert_cuts_for( - self, original_track_ids: frozenset[TrackId] - ) -> "PolarsTrackDataset": + def revert_cuts_for(self, original_track_ids: TrackIdSet) -> "PolarsTrackDataset": if self._dataset.is_empty(): return self @@ -721,7 +721,7 @@ def revert_cuts_for( calculator=self.calculator, ) - def _get_existing_track_ids(self, track_ids: frozenset[TrackId]) -> list[str]: + def _get_existing_track_ids(self, track_ids: TrackIdSet) -> TrackIdSet: if self._dataset.is_empty(): return [] @@ -874,51 +874,3 @@ def area_section_to_shapely(section: Section) -> BaseGeometry: geometry = shapely.Polygon([(c.x, c.y) for c in section.get_coordinates()]) prepare(geometry) return geometry - - -class PolarsTrackIdSet(TrackIdSet): - """ - Polars-based implementation of TrackIdSet using a Series for optimal performance. - """ - - def __init__(self, track_ids: Iterable[TrackId] | pl.Series | None = None): - if track_ids is None: - self._series = pl.Series([], dtype=pl.String) - elif isinstance(track_ids, pl.Series): - self._series = track_ids.unique().sort() - else: - # Convert TrackId objects to strings - id_strings = [unpack(track_id) for track_id in track_ids] - self._series = pl.Series(id_strings, dtype=pl.String).unique().sort() - - def __iter__(self) -> Iterator[TrackId]: - for id_str in self._series: - yield TrackId(id_str) - - def __len__(self) -> int: - return len(self._series) - - def intersection(self, other: "TrackIdSet") -> "TrackIdSet": - if isinstance(other, PolarsTrackIdSet): - # Efficient intersection using Polars operations - intersected_series = self._series.filter(self._series.is_in(other._series)) - return PolarsTrackIdSet(intersected_series) - else: - # Convert other to set of strings for intersection - other_strings = {unpack(track_id) for track_id in other} - intersected_series = self._series.filter( - self._series.is_in(list(other_strings)) - ) - return PolarsTrackIdSet(intersected_series) - - def union(self, other: "TrackIdSet") -> "TrackIdSet": - if isinstance(other, PolarsTrackIdSet): - # Efficient union using Polars operations - combined_series = pl.concat([self._series, other._series]).unique().sort() - return PolarsTrackIdSet(combined_series) - else: - # Convert other to strings and combine - other_strings = [unpack(track_id) for track_id in other] - other_series = pl.Series(other_strings, dtype=pl.String) - combined_series = pl.concat([self._series, other_series]).unique().sort() - return PolarsTrackIdSet(combined_series) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index e4de7ca97..28d1d0104 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime from math import ceil -from typing import Callable, Iterable, Iterator, Optional, Sequence +from typing import Callable, Iterable, Iterator, Optional, Sequence, Any from more_itertools import batched @@ -42,6 +42,7 @@ TrackGeometryDataset, TrackIdSet, TrackSegmentDataset, + contains_true, ) from OTAnalytics.plugin_intersect.shapely.mapping import ShapelyMapper @@ -619,6 +620,21 @@ def contained_by_sections( geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.contained_by_sections(sections) + def ids_inside(self, sections: list[Section]) -> TrackIdSet: + results: set[TrackId] = set() + for cutting_section in sections: + offset = cutting_section.get_offset(EventType.SECTION_ENTER) + results.update( + set( + track_id + for track_id, section_data in ( + self.contained_by_sections([cutting_section], offset).items() + ) + if contains_true(section_data) + ) + ) + return PythonTrackIdSet(results) + def calculate_geometries_for( self, offsets: Iterable[RelativeOffsetCoordinate] ) -> None: @@ -950,6 +966,13 @@ def __init__(self, track_ids: Iterable[TrackId] | None = None): else: self._ids = set(track_ids) + def __eq__(self, other: Any) -> bool: + if isinstance(other, PythonTrackIdSet): + return self._ids == other._ids + elif isinstance(other, TrackIdSet): + return self._ids == PythonTrackIdSet(other)._ids + return False + def __iter__(self) -> Iterator[TrackId]: return iter(self._ids) @@ -967,3 +990,9 @@ def union(self, other: "TrackIdSet") -> "TrackIdSet": return PythonTrackIdSet(self._ids | other._ids) else: return PythonTrackIdSet(self._ids | set(other)) + + def difference(self, other: "TrackIdSet") -> "TrackIdSet": + if isinstance(other, PythonTrackIdSet): + return PythonTrackIdSet(self._ids - other._ids) + else: + return PythonTrackIdSet(self._ids - set(other)) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 410844e08..33f76a125 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -13,8 +13,10 @@ IntersectionPoint, TrackDataset, TrackGeometryDataset, + TrackIdSet, ) from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSet # Column names for track segments ROW_ID = "row_id" @@ -736,7 +738,7 @@ def get_for(self, track_ids: list[str]) -> "PolarsTrackGeometryDataset": return PolarsTrackGeometryDataset(self._offset, filtered_segments) - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: list[Section]) -> TrackIdSet: """Return a set of tracks intersecting a set of sections. Args: @@ -746,7 +748,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: set[TrackId]: the track ids intersecting the given sections. """ if self.empty or not sections: - return set() + return PolarsTrackIdSet() # Create a set to store the track IDs that intersect with any section intersecting_track_ids: set[TrackId] = set() diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index ea929624e..335708aaf 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -37,7 +37,7 @@ TrackDoesNotExistError, TrackGeometryDataset, TrackIdSet, - TrackSegmentDataset, + TrackSegmentDataset, contains_true, ) from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_parser import ottrk_dataformat @@ -530,6 +530,23 @@ def contained_by_sections( geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.contained_by_sections(sections) + def ids_inside(self, sections: list[Section]) -> TrackIdSet: + results: set[TrackId] = set() + for cutting_section in sections: + offset = cutting_section.get_offset(EventType.SECTION_ENTER) + # set of all tracks where at least one coordinate is contained + # by at least one cutting section + results.update( + set( + track_id + for track_id, section_data in ( + self.contained_by_sections([cutting_section], offset).items() + ) + if contains_true(section_data) + ) + ) + return PythonTrackIdSet(results) + def calculate_geometries_for( self, offsets: Iterable[RelativeOffsetCoordinate] ) -> None: diff --git a/OTAnalytics/plugin_filter/pandas_track_id.py b/OTAnalytics/plugin_filter/pandas_track_id.py index 11f511441..c8a076599 100644 --- a/OTAnalytics/plugin_filter/pandas_track_id.py +++ b/OTAnalytics/plugin_filter/pandas_track_id.py @@ -1,6 +1,6 @@ -from typing import Iterable - -from OTAnalytics.domain.track import TRACK_ID, TrackId, TrackIdProvider +from OTAnalytics.domain.track import TRACK_ID, TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.plugin_datastore.track_store import PandasDataFrameProvider @@ -8,7 +8,7 @@ class PandasTrackIdProvider(TrackIdProvider): def __init__(self, pandas_data_frame_provider: PandasDataFrameProvider) -> None: self._pandas_data_frame_provider = pandas_data_frame_provider - def get_ids(self) -> Iterable[TrackId]: + def get_ids(self) -> TrackIdSet: data = self._pandas_data_frame_provider.get_data() if data.empty: diff --git a/OTAnalytics/plugin_number_of_tracks_to_be_validated/svz/number_of_tracks_to_be_validated.py b/OTAnalytics/plugin_number_of_tracks_to_be_validated/svz/number_of_tracks_to_be_validated.py index e10476a41..8d6724d2f 100644 --- a/OTAnalytics/plugin_number_of_tracks_to_be_validated/svz/number_of_tracks_to_be_validated.py +++ b/OTAnalytics/plugin_number_of_tracks_to_be_validated/svz/number_of_tracks_to_be_validated.py @@ -4,12 +4,8 @@ from OTAnalytics.application.use_cases.number_of_tracks_to_be_validated import ( NumberOfTracksToBeValidated, ) -from OTAnalytics.domain.track import ( - OCCURRENCE, - TRACK_CLASSIFICATION, - TRACK_ID, - TrackIdProvider, -) +from OTAnalytics.domain.track import OCCURRENCE, TRACK_CLASSIFICATION, TRACK_ID +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.plugin_number_of_tracks_to_be_validated.calculation_strategy import ( DETECTION_RATE, DetectionRateCalculationStrategy, diff --git a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py index a25a43ad7..a4aea30a1 100644 --- a/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py +++ b/OTAnalytics/plugin_prototypes/track_visualization/track_viz.py @@ -43,13 +43,13 @@ PilImage, Track, TrackId, - TrackIdProvider, TrackImage, W, X, Y, unpack, ) +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import ( TrackListObserver, TrackRepository, diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index 1e21c080c..9878c08b3 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -172,7 +172,7 @@ from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.remark import RemarkRepository from OTAnalytics.domain.section import SectionRepository -from OTAnalytics.domain.track import TrackIdProvider +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository from OTAnalytics.plugin_datastore.polars_track_store import ( diff --git a/OTAnalytics/plugin_ui/gui_application.py b/OTAnalytics/plugin_ui/gui_application.py index 4f22372db..9e29586d2 100644 --- a/OTAnalytics/plugin_ui/gui_application.py +++ b/OTAnalytics/plugin_ui/gui_application.py @@ -13,7 +13,7 @@ SectionProvider, SimpleCreateIntersectionEvents, ) -from OTAnalytics.domain.track import TrackIdProvider +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.plugin_filter.pandas_track_id import PandasTrackIdProvider from OTAnalytics.plugin_prototypes.eventlist_exporter.eventlist_exporter import ( AVAILABLE_EVENTLIST_EXPORTERS, diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 455920393..3493de24e 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -39,7 +39,7 @@ from OTAnalytics.domain.flow import FlowId, FlowRepository from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import SectionId -from OTAnalytics.domain.track import TrackIdProvider +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_filter.dataframe_filter import DataFrameFilterBuilder from OTAnalytics.plugin_intersect.simple_intersect import ( diff --git a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py index a2c0aee8b..55c734c4f 100644 --- a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py +++ b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py @@ -34,7 +34,8 @@ from OTAnalytics.domain.filter import FilterElement from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Detection, Track, TrackId, TrackIdProvider +from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository diff --git a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py index 2d06d56b1..fae51d873 100644 --- a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py +++ b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py @@ -9,7 +9,9 @@ from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.section import SectionId, SectionRepositoryEvent from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet, EmptyTrackIdSet from OTAnalytics.domain.track_repository import TrackRepositoryEvent +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @pytest.fixture @@ -23,8 +25,8 @@ def get_cutting_sections() -> Mock: @pytest.fixture -def track_ids() -> set[TrackId]: - return {TrackId("0"), TrackId("8"), TrackId("15")} +def track_ids() -> TrackIdSet: + return PythonTrackIdSet({TrackId("0"), TrackId("8"), TrackId("15")}) @pytest.fixture @@ -54,10 +56,14 @@ def cached_track_ids_inside_cutting_sections() -> CachedTrackIdsInsideCuttingSec cached_track_ids_inside_cutting_sections = CachedTrackIdsInsideCuttingSections( get_all_tracks, get_cutting_sections ) - cached_track_ids_inside_cutting_sections._cached_ids = { - TrackId("1"), - TrackId("2"), - } + from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet + + cached_track_ids_inside_cutting_sections._cached_ids = PythonTrackIdSet( + { + TrackId("1"), + TrackId("2"), + } + ) return cached_track_ids_inside_cutting_sections @@ -78,7 +84,7 @@ def test__init__(self, get_tracks: Mock, get_cutting_sections: Mock) -> None: def test_get_ids( self, cached_track_ids_inside_cutting_sections: CachedTrackIdsInsideCuttingSections, - track_ids: set[TrackId], + track_ids: TrackIdSet, ) -> None: cached_track_ids_inside_cutting_sections._cached_ids = track_ids assert cached_track_ids_inside_cutting_sections.get_ids() == track_ids @@ -101,7 +107,7 @@ def test_notify_removed_tracks_updates_cache( remove_track_event: TrackRepositoryEvent, ) -> None: cached_track_ids_inside_cutting_sections.notify_tracks(remove_track_event) - assert cached_track_ids_inside_cutting_sections._cached_ids == {TrackId("1")} + assert cached_track_ids_inside_cutting_sections._cached_ids == PythonTrackIdSet({TrackId("1")}) def test_notify_add_sections_clear_cache( self, @@ -124,9 +130,9 @@ def test_notify_remove_sections_clear_cache( ) def test__reset_cache( - self, get_tracks: Mock, get_cutting_sections: Mock, track_ids: set[TrackId] + self, get_tracks: Mock, get_cutting_sections: Mock, track_ids: TrackIdSet ) -> None: cached = CachedTrackIdsInsideCuttingSections(get_tracks, get_cutting_sections) cached._cached_ids = track_ids cached._reset_cache() - assert cached._cached_ids == set() + assert cached._cached_ids == EmptyTrackIdSet() diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 431266ab7..37c1772b0 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -17,9 +17,11 @@ RemoveTracks, TrackRepositorySize, ) -from OTAnalytics.domain.track import Track, TrackId, TrackIdProvider +from OTAnalytics.domain.track import Track, TrackId from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @pytest.fixture @@ -51,7 +53,7 @@ def track_file_repository(track_files: list[Mock]) -> Mock: def create_track_ids_from_ids(ids: Iterable[str]) -> Iterable[TrackId]: - return [TrackId(id) for id in ids] + return PythonTrackIdSet({TrackId(id) for id in ids}) @pytest.fixture diff --git a/tests/OTAnalytics/application/use_cases/test_track_statistics.py b/tests/OTAnalytics/application/use_cases/test_track_statistics.py index f5cc660bc..362259f14 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_statistics.py +++ b/tests/OTAnalytics/application/use_cases/test_track_statistics.py @@ -14,7 +14,8 @@ NumberOfTracksToBeValidated, ) from OTAnalytics.application.use_cases.track_statistics import CalculateTrackStatistics -from OTAnalytics.domain.track import TrackId, TrackIdProvider +from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_id_provider import TrackIdProvider CUTTING_SECTION_NAME: str = "#clicut 0815" NUMBER_OF_TRACKS_TO_BE_VALIDATED = 23 diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index a5b8e1562..5cd72beb2 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -19,9 +19,9 @@ Detection, Track, TrackId, - TrackIdProvider, TrackImage, ) +from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository, TrackRepositoryEvent from OTAnalytics.plugin_datastore.python_track_store import ( PythonDetection, From 5da2e424c1dac2890084730c39553a0f635538bf Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Sat, 30 Aug 2025 14:57:22 +0200 Subject: [PATCH 052/128] Use TrackIdSet where possible --- OTAnalytics/application/analysis/intersect.py | 4 +- .../application/analysis/traffic_counting.py | 5 +-- .../use_cases/highlight_intersections.py | 20 ++++++---- .../application/use_cases/revert_cuts.py | 6 +-- .../use_cases/road_user_assignment_export.py | 7 +++- .../track_dataset/filtered_track_dataset.py | 4 +- .../domain/track_dataset/track_dataset.py | 19 ++++++---- OTAnalytics/domain/track_repository.py | 15 +++++--- .../plugin_datastore/polars_track_store.py | 20 ++++++---- .../plugin_datastore/python_track_store.py | 27 ++++++------- .../pandas_geometry_store.py | 18 ++++++--- .../polars_geometry_store.py | 12 ++++-- .../track_geometry_store/shapely_store.py | 6 ++- OTAnalytics/plugin_datastore/track_store.py | 38 ++++++++++--------- OTAnalytics/plugin_filter/pandas_track_id.py | 9 +++-- .../plugin_intersect/simple_intersect.py | 18 ++++----- .../plugin_ui/intersection_repository.py | 21 +++++----- convert_ottrk_to_feather.py | 2 +- tests/OTAnalytics/application/test_state.py | 3 +- .../use_cases/test_highlight_intersections.py | 37 +++++++++++------- .../use_cases/test_inside_cutting_section.py | 8 ++-- .../application/use_cases/test_revert_cuts.py | 3 +- .../use_cases/test_track_repository.py | 4 +- .../domain/test_track_repository.py | 3 +- .../test_filtered_track_dataset.py | 7 ++-- .../track_dataset/test_track_dataset.py | 5 ++- .../test_polars_track_store.py | 3 +- .../test_python_track_store.py | 16 +++++--- .../plugin_datastore/test_track_store.py | 23 +++++++---- .../test_geometry_store_comparison.py | 6 +-- .../test_shapely_store.py | 3 +- .../plugin_parser/test_streaming_parser.py | 3 +- .../plugin_ui/test_intersection_repository.py | 22 ++++++++--- tests/conftest.py | 12 ++++-- 34 files changed, 248 insertions(+), 161 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index bbd1206e4..0d98e7b2e 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -5,7 +5,7 @@ from OTAnalytics.domain.event import Event from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet from OTAnalytics.domain.types import EventType @@ -26,7 +26,7 @@ def __call__(self, sections: Iterable[Section]) -> list[Event]: class TracksIntersectingSections(ABC): @abstractmethod - def __call__(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]]: + def __call__(self, sections: Iterable[Section]) -> dict[SectionId, TrackIdSet]: raise NotImplementedError diff --git a/OTAnalytics/application/analysis/traffic_counting.py b/OTAnalytics/application/analysis/traffic_counting.py index f382d40e5..51680f232 100644 --- a/OTAnalytics/application/analysis/traffic_counting.py +++ b/OTAnalytics/application/analysis/traffic_counting.py @@ -18,7 +18,6 @@ from OTAnalytics.domain.flow import Flow, FlowRepository from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.types import EventType -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet LEVEL_FROM_SECTION = "from section" LEVEL_TO_SECTION = "to section" @@ -598,9 +597,7 @@ def road_user_ids(self) -> list[str]: Returns: list[str]: the road user ids. """ - return PythonTrackIdSet( - sorted([assignment.road_user for assignment in self._assignments]) - ) + return sorted([assignment.road_user for assignment in self._assignments]) def __init__(self, assignments: list[RoadUserAssignment]) -> None: self._assignments = assignments.copy() diff --git a/OTAnalytics/application/use_cases/highlight_intersections.py b/OTAnalytics/application/use_cases/highlight_intersections.py index 453cf0ad5..d1e686a47 100644 --- a/OTAnalytics/application/use_cases/highlight_intersections.py +++ b/OTAnalytics/application/use_cases/highlight_intersections.py @@ -17,13 +17,14 @@ from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class IntersectionRepository(ABC): - def store(self, intersections: dict[SectionId, set[TrackId]]) -> None: + def store(self, intersections: dict[SectionId, TrackIdSet]) -> None: raise NotImplementedError - def get(self, sections: set[SectionId]) -> dict[SectionId, set[TrackId]]: + def get(self, sections: set[SectionId]) -> dict[SectionId, TrackIdSet]: raise NotImplementedError def clear(self) -> None: @@ -161,7 +162,10 @@ def get_ids(self) -> TrackIdSet: existing_intersections = self._intersection_repository.get(self._section_ids) section_ids_to_process = self._section_ids - existing_intersections.keys() new_intersections = self._calculate_new_intersections(section_ids_to_process) - return new_intersections.union(*existing_intersections.values()) + result = new_intersections + for track_id_set in existing_intersections.values(): + result = result.union(track_id_set) + return result def _calculate_new_intersections( self, section_ids_to_process: set[SectionId] @@ -172,7 +176,7 @@ def _calculate_new_intersections( intersections = self._tracks_intersecting_sections(sections) self._intersection_repository.store(intersections) new_intersections.update(*intersections.values()) - return new_intersections + return PythonTrackIdSet(new_intersections) class TracksNotIntersectingSelection(TrackIdProvider): @@ -194,7 +198,7 @@ def __init__( def get_ids(self) -> TrackIdSet: all_track_ids = {track.id for track in self._track_repository.get_all()} assigned_tracks = set(self._track_id_provider.get_ids()) - return all_track_ids - assigned_tracks + return PythonTrackIdSet(all_track_ids - assigned_tracks) class TracksAssignedToSelectedFlows(TrackIdProvider): @@ -231,7 +235,7 @@ def get_ids(self) -> TrackIdSet: if assignment.assignment.id in self._flow_state.selected_flows.get(): ids.add(TrackId(assignment.road_user)) print(f"Tracks assigned to selected flow(s): {len(ids)}") - return ids + return PythonTrackIdSet(ids) class TracksAssignedToAllFlows(TrackIdProvider): @@ -293,7 +297,7 @@ def get_ids(self) -> TrackIdSet: for assignment in assignments: if assignment.assignment.id in self._flows: ids.add(TrackId(assignment.road_user)) - return ids + return PythonTrackIdSet(ids) class TracksOverlapOccurrenceWindow(TrackIdProvider): @@ -322,7 +326,7 @@ def get_ids(self) -> TrackIdSet: else: tracks = self._track_repository.get_all() - return self._filter(tracks) + return PythonTrackIdSet(self._filter(tracks)) def _get_other_track_ids(self) -> Iterable[Track]: if self._other: diff --git a/OTAnalytics/application/use_cases/revert_cuts.py b/OTAnalytics/application/use_cases/revert_cuts.py index 3cc8c02eb..d748e980f 100644 --- a/OTAnalytics/application/use_cases/revert_cuts.py +++ b/OTAnalytics/application/use_cases/revert_cuts.py @@ -1,4 +1,4 @@ -from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet from OTAnalytics.domain.track_repository import TrackRepository @@ -15,14 +15,14 @@ class RevertCuts: def __init__(self, track_repository: TrackRepository) -> None: self._track_repository = track_repository - def revert(self, original_ids: frozenset[TrackId]) -> None: + def revert(self, original_ids: TrackIdSet) -> None: """Revert cuts for tracks with the specified original track IDs. This method reverts cuts for tracks identified by their original IDs by delegating to the track repository. Args: - original_ids (frozenset[TrackId]): original track IDs for which cuts should + original_ids (TrackIdSet): original track IDs for which cuts should be reverted. """ self._track_repository.revert_cuts_for(original_ids) diff --git a/OTAnalytics/application/use_cases/road_user_assignment_export.py b/OTAnalytics/application/use_cases/road_user_assignment_export.py index f11fdfc77..79e989c0b 100644 --- a/OTAnalytics/application/use_cases/road_user_assignment_export.py +++ b/OTAnalytics/application/use_cases/road_user_assignment_export.py @@ -16,6 +16,8 @@ from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.flow import FlowRepository from OTAnalytics.domain.section import Section, SectionId, SectionRepository +from OTAnalytics.domain.track import TrackId +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet MaxConfidenceLookupTable = dict[str, float] MaxConfidenceProvider = Callable[[list[str]], MaxConfidenceLookupTable] @@ -196,9 +198,10 @@ def _convert(self, assignments: RoadUserAssignments) -> list[dict]: def _get_max_conf_lookup_table_for( self, assignments: RoadUserAssignments ) -> MaxConfidenceLookupTable: - return self._get_all_tracks.as_dataset().get_max_confidences_for( - assignments.road_user_ids + track_id_set = PythonTrackIdSet( + [TrackId(track_id) for track_id in assignments.road_user_ids] ) + return self._get_all_tracks.as_dataset().get_max_confidences_for(track_id_set) def _get_section_by_id(self, section_id: SectionId) -> Section: result = self._section_repository.get(section_id) diff --git a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py index 9f60784a9..6c781f928 100644 --- a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py +++ b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py @@ -52,7 +52,7 @@ def as_list(self) -> list[Track]: def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: + ) -> TrackIdSet: return self._filter().intersecting_tracks(sections, offset) def intersection_points( @@ -74,7 +74,7 @@ def get_first_segments(self) -> TrackSegmentDataset: def get_last_segments(self) -> TrackSegmentDataset: return self._filter().get_last_segments() - def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: + def get_max_confidences_for(self, track_ids: TrackIdSet) -> dict[str, float]: return self._filter().get_max_confidences_for(track_ids) diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index c5ba537dd..75eb1ac0d 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -6,7 +6,6 @@ from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Track, TrackId -from OTAnalytics.domain.types import EventType START_X: str = "start_x" START_Y: str = "start_y" @@ -94,17 +93,23 @@ def difference(self, other: "TrackIdSet") -> "TrackIdSet": class EmptyTrackIdSet(TrackIdSet): def __eq__(self, other: Any) -> bool: return isinstance(other, EmptyTrackIdSet) + def __iter__(self) -> Iterator[TrackId]: return iter([]) + def __len__(self) -> int: return 0 + def intersection(self, other: "TrackIdSet") -> "TrackIdSet": - return other - def union(self, other: "TrackIdSet") -> "TrackIdSet": return self + + def union(self, other: "TrackIdSet") -> "TrackIdSet": + return other + def difference(self, other: "TrackIdSet") -> "TrackIdSet": return self + class TrackDataset(ABC): @property @abstractmethod @@ -258,7 +263,7 @@ def get_last_segments(self) -> TrackSegmentDataset: @abstractmethod def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple["TrackDataset", TrackId]: + ) -> tuple["TrackDataset", TrackIdSet]: """Use section to cut track with TrackDataset. Args: @@ -266,7 +271,7 @@ def cut_with_section( offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. Returns: - tuple[TrackDataset, set[TrackId]]: the dataset containing the cut tracks + tuple[TrackDataset, TrackIdSet]: the dataset containing the cut tracks and the original track ids that have been cut. """ raise NotImplementedError @@ -384,14 +389,14 @@ def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": raise NotImplementedError @abstractmethod - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: list[Section]) -> TrackIdSet: """Return a set of tracks intersecting a set of sections. Args: sections (list[Section]): the list of sections to intersect. Returns: - set[TrackId]: the track ids intersecting the given sections. + TrackIdSet: the track ids intersecting the given sections. """ raise NotImplementedError diff --git a/OTAnalytics/domain/track_repository.py b/OTAnalytics/domain/track_repository.py index 975096c70..8422f7f15 100644 --- a/OTAnalytics/domain/track_repository.py +++ b/OTAnalytics/domain/track_repository.py @@ -7,7 +7,12 @@ from OTAnalytics.application.logger import logger from OTAnalytics.domain.observer import OBSERVER, Subject from OTAnalytics.domain.track import Track, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset, TrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import ( + EmptyTrackIdSet, + TrackDataset, + TrackIdSet, +) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @dataclass(frozen=True) @@ -17,11 +22,11 @@ class TrackRepositoryEvent: @staticmethod def create_added(tracks: Iterable[TrackId]) -> "TrackRepositoryEvent": - return TrackRepositoryEvent(frozenset(tracks), frozenset()) + return TrackRepositoryEvent(PythonTrackIdSet(tracks), EmptyTrackIdSet()) @staticmethod def create_removed(tracks: Iterable[TrackId]) -> "TrackRepositoryEvent": - return TrackRepositoryEvent(frozenset(), frozenset(tracks)) + return TrackRepositoryEvent(EmptyTrackIdSet(), PythonTrackIdSet(tracks)) class TrackListObserver(ABC): @@ -221,7 +226,7 @@ def clear(self) -> None: def __len__(self) -> int: return len(self._dataset) - def revert_cuts_for(self, original_ids: frozenset[TrackId]) -> None: + def revert_cuts_for(self, original_ids: TrackIdSet) -> None: """ Reverts cuts in the dataset for the provided set of original IDs. @@ -231,7 +236,7 @@ def revert_cuts_for(self, original_ids: frozenset[TrackId]) -> None: cuts for the specified IDs. Args: - original_ids (frozenset[TrackId]): A set of strings representing the + original_ids (TrackIdSet): A set of TrackId representing the original IDs of the items in the dataset for which the cuts need to be reverted. diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 0e2532ae2..8e89bbfa0 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -615,10 +615,12 @@ def __create_segments(self) -> pl.DataFrame: def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple["TrackDataset", set[TrackId]]: + ) -> tuple["TrackDataset", TrackIdSet]: if len(self) == 0: logger().info("No tracks to cut") - return self, set() + from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet + + return self, EmptyTrackIdSet() geometry_dataset = self._get_geometry_dataset_for(offset) new_track_ids = geometry_dataset.track_ids_after_cut(section) @@ -628,7 +630,7 @@ def cut_with_section( result = result.drop("old_track_id") return ( PolarsTrackDataset.from_dataframe(result, self.track_geometry_factory), - set(), + PolarsTrackIdSet(), ) def _add_cut_track_part( @@ -676,8 +678,9 @@ def get_max_confidences_for(self, track_ids: TrackIdSet) -> dict[str, float]: return {} try: + track_id_strings = [track_id.id for track_id in track_ids] result = ( - self._dataset.filter(pl.col(LEVEL_TRACK_ID).is_in(track_ids)) + self._dataset.filter(pl.col(LEVEL_TRACK_ID).is_in(track_id_strings)) .group_by(LEVEL_TRACK_ID) .agg(pl.col(track.CONFIDENCE).max()) ) @@ -707,8 +710,9 @@ def revert_cuts_for(self, original_track_ids: TrackIdSet) -> "PolarsTrackDataset return self # Revert track IDs back to original IDs + ids_to_revert_strings = [track_id.id for track_id in ids_to_revert] result = self._dataset.with_columns( - pl.when(pl.col(LEVEL_TRACK_ID).is_in(ids_to_revert)) + pl.when(pl.col(LEVEL_TRACK_ID).is_in(ids_to_revert_strings)) .then(pl.col(track.ORIGINAL_TRACK_ID)) .otherwise(pl.col(LEVEL_TRACK_ID)) .alias(LEVEL_TRACK_ID) @@ -723,7 +727,9 @@ def revert_cuts_for(self, original_track_ids: TrackIdSet) -> "PolarsTrackDataset def _get_existing_track_ids(self, track_ids: TrackIdSet) -> TrackIdSet: if self._dataset.is_empty(): - return [] + from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet + + return EmptyTrackIdSet() converted_ids = [track_id.id for track_id in track_ids] existing_ids = ( @@ -732,7 +738,7 @@ def _get_existing_track_ids(self, track_ids: TrackIdSet) -> TrackIdSet: .unique() .to_list() ) - return existing_ids + return PolarsTrackIdSet([TrackId(track_id) for track_id in existing_ids]) def split(self, batches: int) -> Sequence["PolarsTrackDataset"]: dataset_size = len(self) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 28d1d0104..7b2280e93 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime from math import ceil -from typing import Callable, Iterable, Iterator, Optional, Sequence, Any +from typing import Any, Callable, Iterable, Iterator, Optional, Sequence from more_itertools import batched @@ -36,6 +36,7 @@ START_X, START_Y, TRACK_GEOMETRY_FACTORY, + EmptyTrackIdSet, IntersectionPoint, TrackDataset, TrackDoesNotExistError, @@ -44,6 +45,7 @@ TrackSegmentDataset, contains_true, ) +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_intersect.shapely.mapping import ShapelyMapper @@ -583,9 +585,10 @@ def filter_by_min_detection_length(self, length: int) -> "PythonTrackDataset": def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: + ) -> TrackIdSet: geometry_dataset = self._get_geometry_dataset_for(offset) - return geometry_dataset.intersecting_tracks(sections) + intersecting_ids = geometry_dataset.intersecting_tracks(sections) + return PythonTrackIdSet(intersecting_ids) def _get_geometry_dataset_for( self, offset: RelativeOffsetCoordinate @@ -666,10 +669,10 @@ def __create_last_segment(track: Track) -> PythonTrackSegment: def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple["PythonTrackDataset", set[TrackId]]: + ) -> tuple["PythonTrackDataset", TrackIdSet]: if len(self) == 0: logger().info("No tracks to cut") - return self, set() + return self, EmptyTrackIdSet() shapely_mapper = ShapelyMapper() intersecting_track_ids = self.intersecting_tracks([section], offset) @@ -686,22 +689,20 @@ def cut_with_section( intersecting_track_ids, ) - def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: + def get_max_confidences_for(self, track_ids: TrackIdSet) -> dict[str, float]: result: dict[str, float] = {} for track_id in track_ids: - _track = self.get_for(TrackId(track_id)) + _track = self.get_for(track_id) if not _track: raise TrackDoesNotExistError(f"Track {track_id} not found.") max_confidence = max( [detection.confidence for detection in _track.detections] ) - result[track_id] = max_confidence + result[track_id.id] = max_confidence return result - def revert_cuts_for( - self, original_track_ids: frozenset[TrackId] - ) -> "PythonTrackDataset": + def revert_cuts_for(self, original_track_ids: TrackIdSet) -> "PythonTrackDataset": # NOTE: This implementation prioritizes maintainability over performance. # If performance becomes a concern in high-volume operations, consider # implementing a mapping cache of original track IDs to their derived segments. @@ -868,11 +869,11 @@ def calculate_geometries_for( def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple["TrackDataset", set[TrackId]]: + ) -> tuple["TrackDataset", TrackIdSet]: dataset, original_track_ids = self._other.cut_with_section(section, offset) return self.wrap(dataset), original_track_ids - def revert_cuts_for(self, original_track_ids: frozenset[TrackId]) -> TrackDataset: + def revert_cuts_for(self, original_track_ids: TrackIdSet) -> TrackDataset: return self.wrap(self._other.revert_cuts_for(original_track_ids)) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py index 2c0b2dbfa..b2361d944 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py @@ -18,8 +18,10 @@ IntersectionPoint, TrackDataset, TrackGeometryDataset, + TrackIdSet, ) from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset # Column names for track points @@ -756,17 +758,17 @@ def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": return PandasTrackGeometryDataset(self._offset, filtered_segments) - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: list[Section]) -> TrackIdSet: """Return a set of tracks intersecting a set of sections. Args: sections (list[Section]): the list of sections to intersect. Returns: - set[TrackId]: the track ids intersecting the given sections. + TrackIdSet: the track ids intersecting the given sections. """ if self.empty or not sections: - return set() + return PythonTrackIdSet() # Create a set to store the track IDs that intersect with any section intersecting_track_ids: set[TrackId] = set() @@ -801,7 +803,8 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: # Add track IDs that intersect with the line to the result set intersecting_segments = intersections[intersections[INTERSECTS]] intersecting_track_ids.update( - intersecting_segments[TRACK_ID].unique() + TrackId(track_id) + for track_id in intersecting_segments[TRACK_ID].unique() ) elif section.get_type() == SectionType.AREA: # For area sections, check if any track segment intersects with the @@ -815,9 +818,12 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: # Add track IDs that intersect with the polygon to the result set intersecting_segments = intersections[intersections[INTERSECTS_POLYGON]] - intersecting_track_ids.update(intersecting_segments[TRACK_ID].unique()) + intersecting_track_ids.update( + TrackId(track_id) + for track_id in intersecting_segments[TRACK_ID].unique() + ) - return intersecting_track_ids + return PythonTrackIdSet(intersecting_track_ids) def intersection_points( self, sections: list[Section] diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 33f76a125..05f1733dd 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -784,7 +784,10 @@ def intersecting_tracks(self, sections: list[Section]) -> TrackIdSet: intersecting_segments = intersections.filter(pl.col(INTERSECTS)) if not intersecting_segments.is_empty(): intersecting_track_ids.update( - intersecting_segments[TRACK_ID].unique().to_list() + TrackId(track_id) + for track_id in intersecting_segments[TRACK_ID] + .unique() + .to_list() ) elif section.get_type() == SectionType.AREA: # For area sections, check if any track segment intersects with the @@ -800,10 +803,13 @@ def intersecting_tracks(self, sections: list[Section]) -> TrackIdSet: intersecting_segments = intersections.filter(pl.col(INTERSECTS_POLYGON)) if not intersecting_segments.is_empty(): intersecting_track_ids.update( - intersecting_segments[TRACK_ID].unique().to_list() + TrackId(track_id) + for track_id in intersecting_segments[TRACK_ID] + .unique() + .to_list() ) - return intersecting_track_ids + return PolarsTrackIdSet(intersecting_track_ids) def intersection_points( self, sections: list[Section] diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py index 72140781d..d1fc36f91 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/shapely_store.py @@ -27,7 +27,9 @@ IntersectionPoint, TrackDataset, TrackGeometryDataset, + TrackIdSet, ) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_datastore.track_store import LEVEL_TRACK_ID, PandasTrackDataset TRACK_ID = "track_id" @@ -241,7 +243,7 @@ def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": filtered_df = self._dataset.loc[_ids] return ShapelyTrackGeometryDataset(self.offset, filtered_df) - def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: + def intersecting_tracks(self, sections: list[Section]) -> TrackIdSet: intersecting_tracks = set() section_geoms = line_sections_to_shapely_multi(sections) prepared_function = partial(calculate_intersects, section_geoms=section_geoms) @@ -254,7 +256,7 @@ def intersecting_tracks(self, sections: list[Section]) -> set[TrackId]: ] intersecting_tracks.update(track_ids) - return intersecting_tracks + return PythonTrackIdSet(intersecting_tracks) def intersection_points( self, sections: list[Section] diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 335708aaf..9be5a3a3e 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -32,13 +32,16 @@ START_X, START_Y, TRACK_GEOMETRY_FACTORY, + EmptyTrackIdSet, IntersectionPoint, TrackDataset, TrackDoesNotExistError, TrackGeometryDataset, TrackIdSet, - TrackSegmentDataset, contains_true, + TrackSegmentDataset, + contains_true, ) +from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_parser import ottrk_dataformat @@ -401,6 +404,7 @@ def remove(self, track_id: TrackId) -> "PandasTrackDataset": ) def remove_multiple(self, track_ids: set[TrackId]) -> "PandasTrackDataset": + # TODO change to TrackIdSet track_ids_primitive = [unpack(track_id) for track_id in track_ids] remaining_tracks = self._dataset.drop(track_ids_primitive, errors="ignore") updated_geometry_datasets = self._remove_from_geometry_dataset( @@ -493,9 +497,10 @@ def filter_by_min_detection_length(self, length: int) -> TrackDataset: def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> set[TrackId]: + ) -> TrackIdSet: geometry_dataset = self._get_geometry_dataset_for(offset) - return geometry_dataset.intersecting_tracks(sections) + intersecting_ids = geometry_dataset.intersecting_tracks(sections) + return PythonTrackIdSet(intersecting_ids) def _get_geometry_dataset_for( self, offset: RelativeOffsetCoordinate @@ -614,10 +619,10 @@ def get_last_segments(self) -> TrackSegmentDataset: def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple["PandasTrackDataset", set[TrackId]]: + ) -> tuple["PandasTrackDataset", TrackIdSet]: if len(self) == 0: logger().info("No tracks to cut") - return self, set() + return self, EmptyTrackIdSet() # TODO is this only: # - groupby trackid @@ -647,9 +652,9 @@ def cut_with_section( cut_tracks_df.index = MultiIndex.from_frame( index_as_df[[track.TRACK_ID, track.OCCURRENCE]] ) - return PandasTrackDataset(self.track_geometry_factory, cut_tracks_df), set( - intersection_points.keys() - ) + return PandasTrackDataset( + self.track_geometry_factory, cut_tracks_df + ), PythonTrackIdSet(intersection_points.keys()) def _create_cut_track_id(self, row: Series, cut_info: dict[str, list[int]]) -> str: if (track_id := row[track.TRACK_ID]) in cut_info.keys(): @@ -657,10 +662,11 @@ def _create_cut_track_id(self, row: Series, cut_info: dict[str, list[int]]) -> s return f"{track_id}_{cut_segment_index}" return row[track.TRACK_ID] - def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: + def get_max_confidences_for(self, track_ids: TrackIdSet) -> dict[str, float]: + track_id_strings = [track_id.id for track_id in track_ids] try: return ( - self._dataset.loc[track_ids][track.CONFIDENCE] + self._dataset.loc[track_id_strings][track.CONFIDENCE] .groupby(level=[LEVEL_TRACK_ID]) .max() .to_dict() @@ -670,9 +676,7 @@ def get_max_confidences_for(self, track_ids: list[str]) -> dict[str, float]: "Some tracks do not exists in dataset with given id" ) from cause - def revert_cuts_for( - self, original_track_ids: frozenset[TrackId] - ) -> "PandasTrackDataset": + def revert_cuts_for(self, original_track_ids: TrackIdSet) -> "PandasTrackDataset": if self._dataset.empty: return self ids_to_revert = self._get_existing_track_ids(original_track_ids) @@ -690,7 +694,7 @@ def revert_cuts_for( calculator=self.calculator, ) - def _get_existing_track_ids(self, track_ids: frozenset[TrackId]) -> list[str]: + def _get_existing_track_ids(self, track_ids: TrackIdSet) -> list[str]: converted_ids = [track_id.id for track_id in track_ids] return list( self._dataset.loc[ @@ -731,7 +735,7 @@ def calculate_geometries_for( def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple[PandasTrackDataset, set[TrackId]]: + ) -> tuple[PandasTrackDataset, TrackIdSet]: dataset, original_track_ids = self._other.cut_with_section(section, offset) return self.wrap(dataset), original_track_ids @@ -746,9 +750,7 @@ def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: def get_data(self) -> DataFrame: return self._filter().get_data() - def revert_cuts_for( - self, original_track_ids: frozenset[TrackId] - ) -> PandasTrackDataset: + def revert_cuts_for(self, original_track_ids: TrackIdSet) -> PandasTrackDataset: return self.wrap(self._other.revert_cuts_for(original_track_ids)) diff --git a/OTAnalytics/plugin_filter/pandas_track_id.py b/OTAnalytics/plugin_filter/pandas_track_id.py index c8a076599..4b4bf8ef0 100644 --- a/OTAnalytics/plugin_filter/pandas_track_id.py +++ b/OTAnalytics/plugin_filter/pandas_track_id.py @@ -1,6 +1,7 @@ from OTAnalytics.domain.track import TRACK_ID, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackIdSet from OTAnalytics.domain.track_id_provider import TrackIdProvider +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_datastore.track_store import PandasDataFrameProvider @@ -12,7 +13,7 @@ def get_ids(self) -> TrackIdSet: data = self._pandas_data_frame_provider.get_data() if data.empty: - return [] + return EmptyTrackIdSet() if TRACK_ID not in list(data.index.names): raise ValueError( @@ -20,4 +21,6 @@ def get_ids(self) -> TrackIdSet: "must be in the index of DataFrame for retrieving all track ids." ) - return [TrackId(id) for id in data.index.get_level_values(TRACK_ID)] + return PythonTrackIdSet( + [TrackId(id) for id in data.index.get_level_values(TRACK_ID)] + ) diff --git a/OTAnalytics/plugin_intersect/simple_intersect.py b/OTAnalytics/plugin_intersect/simple_intersect.py index d5cb8ad2b..5724a566e 100644 --- a/OTAnalytics/plugin_intersect/simple_intersect.py +++ b/OTAnalytics/plugin_intersect/simple_intersect.py @@ -1,10 +1,9 @@ -from collections import defaultdict from typing import Iterable from OTAnalytics.application.analysis.intersect import TracksIntersectingSections from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet from OTAnalytics.domain.types import EventType @@ -12,21 +11,20 @@ class SimpleTracksIntersectingSections(TracksIntersectingSections): def __init__(self, get_tracks: GetAllTracks): self._get_tracks = get_tracks - def __call__(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]]: + def __call__(self, sections: Iterable[Section]) -> dict[SectionId, TrackIdSet]: return self.intersect(sections) - def intersect(self, sections: Iterable[Section]) -> dict[SectionId, set[TrackId]]: + def intersect(self, sections: Iterable[Section]) -> dict[SectionId, TrackIdSet]: track_dataset = self._get_tracks.as_dataset() - result = defaultdict(set) + result: dict[SectionId, TrackIdSet] = {} total_tracks_intersected = 0 print("Number of intersecting tracks per section") for section in sections: - result[section.id].update( - track_dataset.intersecting_tracks( - [section], section.get_offset(EventType.SECTION_ENTER) - ) + intersecting_tracks = track_dataset.intersecting_tracks( + [section], section.get_offset(EventType.SECTION_ENTER) ) - num_tracks_intersected = len(result[section.id]) + result[section.id] = intersecting_tracks + num_tracks_intersected = len(intersecting_tracks) total_tracks_intersected += num_tracks_intersected print(f"{section.name}: {num_tracks_intersected} tracks") print(f"All sections: {total_tracks_intersected} tracks") diff --git a/OTAnalytics/plugin_ui/intersection_repository.py b/OTAnalytics/plugin_ui/intersection_repository.py index 3ffa60eb4..252f32207 100644 --- a/OTAnalytics/plugin_ui/intersection_repository.py +++ b/OTAnalytics/plugin_ui/intersection_repository.py @@ -1,28 +1,31 @@ -from collections import defaultdict - from OTAnalytics.application.use_cases.highlight_intersections import ( IntersectionRepository, ) from OTAnalytics.domain.section import SectionId -from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet class PythonIntersectionRepository(IntersectionRepository): def __init__(self) -> None: - self._intersections: dict[SectionId, set[TrackId]] = defaultdict(set) + self._intersections: dict[SectionId, TrackIdSet] = {} - def store(self, intersections: dict[SectionId, set[TrackId]]) -> None: + def store(self, intersections: dict[SectionId, TrackIdSet]) -> None: for section, tracks in intersections.items(): - self._intersections[section].update(tracks) - - def get(self, sections: set[SectionId]) -> dict[SectionId, set[TrackId]]: + if section in self._intersections: + self._intersections[section] = self._intersections[section].union( + tracks + ) + else: + self._intersections[section] = tracks + + def get(self, sections: set[SectionId]) -> dict[SectionId, TrackIdSet]: return { section: self._intersections[section] for section in sections if section in self._intersections.keys() } - def get_all(self) -> dict[SectionId, set[TrackId]]: + def get_all(self) -> dict[SectionId, TrackIdSet]: return self._intersections.copy() def clear(self) -> None: diff --git a/convert_ottrk_to_feather.py b/convert_ottrk_to_feather.py index 9a20e98a6..cbfc4d27b 100644 --- a/convert_ottrk_to_feather.py +++ b/convert_ottrk_to_feather.py @@ -34,7 +34,7 @@ from OTAnalytics.plugin_parser.convert_ottrk_to_feathers import convert_ottrk_to_feather -def main(): +def main() -> None: """Main entry point for the script.""" parser = argparse.ArgumentParser( description="Convert ottrk files to Apache Arrow/Feather format with metadata " diff --git a/tests/OTAnalytics/application/test_state.py b/tests/OTAnalytics/application/test_state.py index e08e98fc0..ce53eba31 100644 --- a/tests/OTAnalytics/application/test_state.py +++ b/tests/OTAnalytics/application/test_state.py @@ -33,6 +33,7 @@ SectionType, ) from OTAnalytics.domain.track import Detection, Track, TrackId, TrackImage +from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet from OTAnalytics.domain.track_repository import ( TrackObserver, TrackRepository, @@ -91,7 +92,7 @@ def test_update_selected_track_on_notify_tracks_with_empty_list(self) -> None: state = TrackState() state.notify_tracks(TrackRepositoryEvent.create_added([first_track])) - state.notify_tracks(TrackRepositoryEvent(frozenset(), frozenset())) + state.notify_tracks(TrackRepositoryEvent(EmptyTrackIdSet(), EmptyTrackIdSet())) assert state.selected_track is None diff --git a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py index 55c734c4f..a00b547b5 100644 --- a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py +++ b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py @@ -35,8 +35,10 @@ from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Detection, Track, TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @pytest.fixture @@ -175,7 +177,7 @@ def test_get_ids( track_ids = provider.get_ids() - assert track_ids == {track_id_1, track_id_2} + assert track_ids == PythonTrackIdSet({track_id_1, track_id_2}) class TestTracksIntersectingGivenSections: @@ -189,7 +191,7 @@ def test_get_ids( ) -> None: section_ids = {section_1.id} sections = [section_1] - section_1_tracks = {track_id_1} + section_1_tracks: TrackIdSet = PythonTrackIdSet({track_id_1}) original_track_ids = {section_1.id: section_1_tracks} intersection_repository.get.return_value = {} get_section_by_id.return_value = sections @@ -219,7 +221,7 @@ def test_get_existing_ids( ) -> None: section_ids = {section_1.id} sections = [section_1] - section_1_tracks = {track_id_1} + section_1_tracks: TrackIdSet = PythonTrackIdSet({track_id_1}) original_intersections = {section_1.id: section_1_tracks} intersection_repository.get.return_value = original_intersections get_section_by_id.return_value = sections @@ -252,8 +254,8 @@ def test_get_merged_ids( section_ids = {section_1.id, section_2.id} sections_to_process = [section_2] section_ids_to_process = {section_2.id} - section_1_tracks = {track_id_1} - section_2_tracks = {track_id_2} + section_1_tracks: TrackIdSet = PythonTrackIdSet({track_id_1}) + section_2_tracks: TrackIdSet = PythonTrackIdSet({track_id_2}) existing_intersections = {section_1.id: section_1_tracks} new_intersections = {section_2.id: section_2_tracks} intersection_repository.get.return_value = existing_intersections @@ -268,7 +270,7 @@ def test_get_merged_ids( track_ids = provider.get_ids() - assert track_ids == {track_id_1, track_id_2} + assert track_ids == PythonTrackIdSet({track_id_1, track_id_2}) intersection_repository.get.assert_called_once_with(section_ids) get_section_by_id.assert_called_once_with(section_ids_to_process) tracks_intersecting_sections.assert_called_once_with(sections_to_process) @@ -444,38 +446,45 @@ def test_has_overlap_wrong_order_of_arguments(self) -> None: def test_get_ids(self) -> None: track_repository = Mock(spec=TrackRepository) track_view_state = Mock(spec=TrackViewState) - track_ids = [Mock(spec=TrackId), Mock(spec=TrackId)] + first_id = Mock(spec=TrackId) tracks = [Mock(spec=Track), None] track_repository.get_all.return_value = tracks with patch.object( - TracksOverlapOccurrenceWindow, "_filter", return_value=[track_ids[0]] + TracksOverlapOccurrenceWindow, "_filter", return_value=[first_id] ): id_provider = TracksOverlapOccurrenceWindow( track_repository, track_view_state ) result_ids = id_provider.get_ids() - assert result_ids == [track_ids[0]] + assert result_ids == PythonTrackIdSet([first_id]) track_repository.get_all.assert_called_once() def test_get_ids_as_decorator(self) -> None: track_repository = Mock(spec=TrackRepository) track_view_state = Mock(spec=TrackViewState) - track_ids = [Mock(spec=TrackId), Mock(spec=TrackId)] + first_id = Mock(spec=TrackId) + track_ids = [first_id, Mock(spec=TrackId)] tracks = [Mock(spec=Track), None] track_repository.get_for.side_effect = tracks + other = Mock(spec=TrackIdProvider) + other.get_ids.return_value = track_ids with patch.object( - TracksOverlapOccurrenceWindow, "_filter", return_value=[track_ids[0]] + TracksOverlapOccurrenceWindow, "_filter", return_value=[first_id] ): id_provider = TracksOverlapOccurrenceWindow( - track_repository, track_view_state + track_repository, + track_view_state, + other=other, ) result_ids = id_provider.get_ids() - assert result_ids == [track_ids[0]] - track_repository.get_for.call_args_list == [call(id) for id in track_ids] + assert result_ids == PythonTrackIdSet([first_id]) + assert track_repository.get_for.call_args_list == [ + call(id) for id in track_ids + ] @pytest.mark.parametrize( ( diff --git a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py index fae51d873..909429b34 100644 --- a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py +++ b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py @@ -9,7 +9,7 @@ from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.section import SectionId, SectionRepositoryEvent from OTAnalytics.domain.track import TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet, EmptyTrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackIdSet from OTAnalytics.domain.track_repository import TrackRepositoryEvent from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @@ -79,7 +79,7 @@ def assert__reset_cache_called_once_when_notify_tracks_with( class TestCachedTrackIdsInsideCuttingSections: def test__init__(self, get_tracks: Mock, get_cutting_sections: Mock) -> None: cached = CachedTrackIdsInsideCuttingSections(get_tracks, get_cutting_sections) - assert cached._cached_ids == set() + assert cached._cached_ids == EmptyTrackIdSet() def test_get_ids( self, @@ -107,7 +107,9 @@ def test_notify_removed_tracks_updates_cache( remove_track_event: TrackRepositoryEvent, ) -> None: cached_track_ids_inside_cutting_sections.notify_tracks(remove_track_event) - assert cached_track_ids_inside_cutting_sections._cached_ids == PythonTrackIdSet({TrackId("1")}) + assert cached_track_ids_inside_cutting_sections._cached_ids == PythonTrackIdSet( + {TrackId("1")} + ) def test_notify_add_sections_clear_cache( self, diff --git a/tests/OTAnalytics/application/use_cases/test_revert_cuts.py b/tests/OTAnalytics/application/use_cases/test_revert_cuts.py index 6e9528a35..ba2d21bac 100644 --- a/tests/OTAnalytics/application/use_cases/test_revert_cuts.py +++ b/tests/OTAnalytics/application/use_cases/test_revert_cuts.py @@ -4,11 +4,12 @@ from OTAnalytics.application.use_cases.revert_cuts import RevertCuts from OTAnalytics.domain.track import TrackId +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class TestRevertCuts: def test_revert(self, track_repository: Mock) -> None: - original_ids = frozenset([TrackId("1"), TrackId("2")]) + original_ids = PythonTrackIdSet([TrackId("1"), TrackId("2")]) target = RevertCuts(track_repository) target.revert(original_ids) diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index 37c1772b0..cdd11fbab 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -75,7 +75,7 @@ def test_get_ids_of_filter_by_self(self, track_id_provider: Mock) -> None: ) filtered_ids = filtered_provider.get_ids() - assert filtered_ids == set(track_ids) + assert filtered_ids == PythonTrackIdSet(track_ids) assert track_id_provider.get_ids.call_count == 2 def test_get_ids(self, track_id_provider: Mock) -> None: @@ -90,7 +90,7 @@ def test_get_ids(self, track_id_provider: Mock) -> None: ) filtered_ids = filtered_provider.get_ids() - assert filtered_ids == set(create_track_ids_from_ids(["2", "3"])) + assert filtered_ids == PythonTrackIdSet(create_track_ids_from_ids(["2", "3"])) track_id_provider.get_ids.assert_called_once() filter.get_ids.assert_called_once() diff --git a/tests/OTAnalytics/domain/test_track_repository.py b/tests/OTAnalytics/domain/test_track_repository.py index 94ca4cacb..003c70988 100644 --- a/tests/OTAnalytics/domain/test_track_repository.py +++ b/tests/OTAnalytics/domain/test_track_repository.py @@ -13,6 +13,7 @@ TrackRepositoryEvent, TrackSubject, ) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class TestTrackSubject: @@ -186,7 +187,7 @@ def test_classifications(self) -> None: assert repository.classifications == classifications def test_revert_cuts_for(self) -> None: - original_ids = frozenset([TrackId("1")]) + original_ids = PythonTrackIdSet([TrackId("1")]) dataset = Mock() target = TrackRepository(dataset) target.revert_cuts_for(original_ids) diff --git a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py index 85c481219..78d4253e9 100644 --- a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py +++ b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py @@ -10,6 +10,7 @@ FilterByClassTrackDataset, ) from OTAnalytics.domain.track_dataset.track_dataset import TrackDoesNotExistError +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from tests.utils.assertions import ( assert_equal_track_properties, assert_track_dataset_has_tracks, @@ -67,7 +68,7 @@ def test_track_ids(self, cargo_bike_track: Track, bicycle_track: Track) -> None: [cargo_bike_track, bicycle_track], [OtcClasses.BICYCLIST], [] ) for filtered_dataset in filtered_datasets.values(): - assert filtered_dataset.track_ids == frozenset([bicycle_track.id]) + assert filtered_dataset.track_ids == PythonTrackIdSet([bicycle_track.id]) def test_first_and_last_occurrence( self, car_track: Track, cargo_bike_track: Track @@ -446,10 +447,10 @@ def test_get_max_confidences_for( for empty_dataset in empty_datasets.values(): with pytest.raises(TrackDoesNotExistError): - empty_dataset.get_max_confidences_for([car_track.id.id]) + empty_dataset.get_max_confidences_for(PythonTrackIdSet([car_track.id])) filled_dataset = empty_dataset.add_all([car_track, pedestrian_track]) - all_track_ids = [track_id.id for track_id in filled_dataset.track_ids] + all_track_ids = PythonTrackIdSet(filled_dataset.track_ids) result = filled_dataset.get_max_confidences_for(all_track_ids) assert result == expected diff --git a/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py b/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py index b7cfcc0cb..32f58b46a 100644 --- a/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py +++ b/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py @@ -1,5 +1,6 @@ from OTAnalytics.domain.track import Track from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from tests.utils.assertions import assert_equal_track_properties from tests.utils.builders.track_dataset_provider import ( IMPLEMENTATIONS, @@ -22,7 +23,9 @@ def test_revert_cuts_for( for target in targets: assert len(target) == 4 actual = target.revert_cuts_for( - frozenset([first_track_part_1.original_id, uncut_track.original_id]) + PythonTrackIdSet( + [first_track_part_1.original_id, uncut_track.original_id] + ) ) assert len(actual) == 3 actual_first_track = actual.get_for(first_track_part_1.original_id) diff --git a/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py index d74f1cf3a..9b98bc30f 100644 --- a/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py @@ -4,6 +4,7 @@ from OTAnalytics.domain import track from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSet from OTAnalytics.plugin_datastore.polars_track_store import ( COLUMNS, POLARS_TRACK_GEOMETRY_FACTORY, @@ -345,7 +346,7 @@ def test_track_ids( dataset = PolarsTrackDataset.from_list( [car_track, pedestrian_track], track_geometry_factory ) - expected = frozenset({car_track.id, pedestrian_track.id}) + expected = PolarsTrackIdSet({car_track.id, pedestrian_track.id}) assert dataset.track_ids == expected def test_empty( diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_store.py b/tests/OTAnalytics/plugin_datastore/test_python_track_store.py index 7ac4f2da3..52044e5c0 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_store.py @@ -30,6 +30,7 @@ START_X, START_Y, TRACK_GEOMETRY_FACTORY, + EmptyTrackIdSet, TrackDoesNotExistError, TrackGeometryDataset, ) @@ -38,6 +39,7 @@ PythonDetection, PythonTrack, PythonTrackDataset, + PythonTrackIdSet, PythonTrackSegment, PythonTrackSegmentDataset, SimpleCutTrackPartBuilder, @@ -660,7 +662,7 @@ def test_cut_with_section_no_tracks(self) -> None: Mock(), RelativeOffsetCoordinate(0, 0) ) assert cut_track_dataset == dataset - assert original_track_ids == set() + assert original_track_ids == EmptyTrackIdSet() def test_track_ids( self, @@ -690,14 +692,16 @@ def test_get_max_confidences_for( ) -> None: empty_dataset = PythonTrackDataset(track_geometry_factory) with pytest.raises(TrackDoesNotExistError): - empty_dataset.get_max_confidences_for([car_track.id.id]) + empty_dataset.get_max_confidences_for(PythonTrackIdSet([car_track.id])) filled_dataset = empty_dataset.add_all([car_track, pedestrian_track]) - car_id = car_track.id.id - pedestrian_id = pedestrian_track.id.id + car_id = car_track.id + pedestrian_id = pedestrian_track.id - result = filled_dataset.get_max_confidences_for([car_id, pedestrian_id]) - assert result == {car_id: 0.8, pedestrian_id: 0.9} + result = filled_dataset.get_max_confidences_for( + PythonTrackIdSet([car_id, pedestrian_id]) + ) + assert result == {car_id.id: 0.8, pedestrian_id.id: 0.9} class TestSimpleCutTrackSegmentBuilder: diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 8ba380070..4f47bcefc 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -11,13 +11,16 @@ from OTAnalytics.domain.track import Track, TrackId from OTAnalytics.domain.track_dataset.track_dataset import ( TRACK_GEOMETRY_FACTORY, + EmptyTrackIdSet, TrackDataset, TrackDoesNotExistError, TrackGeometryDataset, + TrackIdSet, ) from OTAnalytics.plugin_datastore.python_track_store import ( PythonTrack, PythonTrackDataset, + PythonTrackIdSet, ) from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, @@ -352,7 +355,9 @@ def test_remove_multiple( first_track = self.__build_track("1") second_track = self.__build_track("2") third_track = self.__build_track("3") - all_track_ids = frozenset([first_track.id, second_track.id, third_track.id]) + all_track_ids = PythonTrackIdSet( + [first_track.id, second_track.id, third_track.id] + ) track_ids_to_remove = {first_track.id, second_track.id} tracks_df = _convert_tracks([first_track, second_track, third_track]) geometry_dataset, updated_geometry_dataset = create_mock_geometry_dataset() @@ -569,7 +574,7 @@ def test_classifications_on_empty_dataset( def test_cut_with_section( self, cutting_section_test_case: tuple[ - LineSection, list[Track], list[Track], set[TrackId] + LineSection, list[Track], list[Track], TrackIdSet ], track_geometry_factory: TRACK_GEOMETRY_FACTORY, ) -> None: @@ -598,7 +603,7 @@ def test_cut_with_section_no_tracks( Mock(), RelativeOffsetCoordinate(0, 0) ) assert cut_track_dataset == dataset - assert original_track_ids == set() + assert original_track_ids == EmptyTrackIdSet() def test_track_ids( self, @@ -629,14 +634,16 @@ def test_get_max_confidences_for( ) -> None: empty_dataset = PandasTrackDataset(track_geometry_factory) with pytest.raises(TrackDoesNotExistError): - empty_dataset.get_max_confidences_for([car_track.id.id]) + empty_dataset.get_max_confidences_for(PythonTrackIdSet([car_track.id])) filled_dataset = empty_dataset.add_all([car_track, pedestrian_track]) - car_id = car_track.id.id - pedestrian_id = pedestrian_track.id.id + car_id = car_track.id + pedestrian_id = pedestrian_track.id - result = filled_dataset.get_max_confidences_for([car_id, pedestrian_id]) - assert result == {car_id: 0.8, pedestrian_id: 0.9} + result = filled_dataset.get_max_confidences_for( + PythonTrackIdSet([car_id, pedestrian_id]) + ) + assert result == {car_id.id: 0.8, pedestrian_id.id: 0.9} def test_create_test_flyweight_with_single_detection( self, track_geometry_factory: TRACK_GEOMETRY_FACTORY diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py index 785b6bf70..6031d057c 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py @@ -169,11 +169,9 @@ def dataframes_equal( def prepare_polars_data_frame( data: dict[str, list[Any]], ) -> pl.DataFrame: - df = pl.DataFrame(data) + df = pl.DataFrame(data).with_row_index(ROW_ID) if OCCURRENCE in df.columns: - return df.with_row_index(ROW_ID).with_columns( - pl.col(OCCURRENCE).dt.cast_time_unit("ns") - ) + return df.with_columns(pl.col(OCCURRENCE).dt.cast_time_unit("ns")) else: return df.with_columns( pl.col(START_OCCURRENCE).dt.cast_time_unit("ns") diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_shapely_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_shapely_store.py index 83b57d413..9ca088c27 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_shapely_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_shapely_store.py @@ -19,6 +19,7 @@ TrackGeometryDataset, ) from OTAnalytics.domain.types import EventType +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( BASE_GEOMETRY, COLUMNS, @@ -645,7 +646,7 @@ def test_intersecting_tracks( track_dataset, BASE_GEOMETRY ) result = geometry_dataset.intersecting_tracks(sections) - assert result == {first_track.id, second_track.id} + assert result == PythonTrackIdSet({first_track.id, second_track.id}) @patch( "OTAnalytics.plugin_datastore.track_geometry_store.shapely_store." diff --git a/tests/OTAnalytics/plugin_parser/test_streaming_parser.py b/tests/OTAnalytics/plugin_parser/test_streaming_parser.py index b02e4a178..a9c0e7a74 100644 --- a/tests/OTAnalytics/plugin_parser/test_streaming_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_streaming_parser.py @@ -13,6 +13,7 @@ from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, PythonTrackDataset, + PythonTrackIdSet, ) from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, @@ -73,7 +74,7 @@ def assert_track_stream_equals_dataset( def assert_track_list_equals_dataset( bulk_res: TrackDataset, tracks: list[Track] ) -> None: - assert bulk_res.track_ids == {track.id for track in tracks} + assert bulk_res.track_ids == PythonTrackIdSet({track.id for track in tracks}) print("Size", len(bulk_res.track_ids), len(tracks)) for actual_track in tracks: diff --git a/tests/OTAnalytics/plugin_ui/test_intersection_repository.py b/tests/OTAnalytics/plugin_ui/test_intersection_repository.py index 60b60055c..ec95d9788 100644 --- a/tests/OTAnalytics/plugin_ui/test_intersection_repository.py +++ b/tests/OTAnalytics/plugin_ui/test_intersection_repository.py @@ -2,6 +2,8 @@ from OTAnalytics.domain.section import SectionId from OTAnalytics.domain.track import TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_ui.intersection_repository import PythonIntersectionRepository @@ -27,7 +29,9 @@ def section_id_2() -> SectionId: class TestPythonIntersectionRepository: def test_store(self, section_id_1: SectionId, track_id_1: TrackId) -> None: - intersections = {section_id_1: {track_id_1}} + intersections: dict[SectionId, TrackIdSet] = { + section_id_1: PythonTrackIdSet({track_id_1}) + } repository = PythonIntersectionRepository() repository.store(intersections) @@ -36,7 +40,9 @@ def test_store(self, section_id_1: SectionId, track_id_1: TrackId) -> None: assert stored_intersections == intersections def test_clear(self, section_id_1: SectionId, track_id_1: TrackId) -> None: - intersections = {section_id_1: {track_id_1}} + intersections: dict[SectionId, TrackIdSet] = { + section_id_1: PythonTrackIdSet({track_id_1}) + } repository = PythonIntersectionRepository() repository.store(intersections) @@ -51,13 +57,16 @@ def test_remove( track_id_1: TrackId, track_id_2: TrackId, ) -> None: - intersections = {section_id_1: {track_id_1}, section_id_2: {track_id_2}} + intersections: dict[SectionId, TrackIdSet] = { + section_id_1: PythonTrackIdSet({track_id_1}), + section_id_2: PythonTrackIdSet({track_id_2}), + } repository = PythonIntersectionRepository() repository.store(intersections) repository.remove({section_id_1}) - assert repository.get_all() == {section_id_2: {track_id_2}} + assert repository.get_all() == {section_id_2: PythonTrackIdSet({track_id_2})} def test_remove_missing(self, section_id_1: SectionId) -> None: repository = PythonIntersectionRepository() @@ -73,7 +82,10 @@ def test_get( track_id_1: TrackId, track_id_2: TrackId, ) -> None: - intersections = {section_id_1: {track_id_1}, section_id_2: {track_id_2}} + intersections: dict[SectionId, TrackIdSet] = { + section_id_1: PythonTrackIdSet({track_id_1}), + section_id_2: PythonTrackIdSet({track_id_2}), + } repository = PythonIntersectionRepository() repository.store(intersections) actual = repository.get({section_id_1, section_id_2}) diff --git a/tests/conftest.py b/tests/conftest.py index 1c727d4b9..67f147d11 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,8 +14,12 @@ from OTAnalytics.domain.geometry import Coordinate from OTAnalytics.domain.otc_classes import OtcClasses from OTAnalytics.domain.section import LineSection, Section, SectionId -from OTAnalytics.domain.track import Track, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY +from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track_dataset.track_dataset import ( + TRACK_GEOMETRY_FACTORY, + TrackIdSet, +) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, ) @@ -270,7 +274,7 @@ def track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: @pytest.fixture def cutting_section_test_case() -> ( - tuple[LineSection, list[Track], list[Track], set[TrackId]] + tuple[LineSection, list[Track], list[Track], TrackIdSet] ): track_id_1 = "1" track_id_2 = "2" @@ -323,7 +327,7 @@ def cutting_section_test_case() -> ( SectionId(_id), _id, {}, {}, [Coordinate(2.5, 0), Coordinate(2.5, 3)] ) - expected_original_track_ids = {first_track.id, second_track.id} + expected_original_track_ids = PythonTrackIdSet({first_track.id, second_track.id}) return ( cutting_section, From 84af6a65cfda6d3ee4f12747b1db85b93f1278cf Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Sat, 30 Aug 2025 15:17:27 +0200 Subject: [PATCH 053/128] Use TrackIdSet to remove multiple tracks at once --- .../application/use_cases/track_repository.py | 3 ++- OTAnalytics/domain/track_dataset/track_dataset.py | 2 +- OTAnalytics/domain/track_repository.py | 4 ++-- OTAnalytics/plugin_datastore/polars_track_store.py | 9 +++++++-- OTAnalytics/plugin_datastore/python_track_store.py | 14 ++++++++------ OTAnalytics/plugin_datastore/track_store.py | 5 ++--- .../application/use_cases/test_track_repository.py | 4 ++-- tests/OTAnalytics/domain/test_track_repository.py | 8 +++++--- .../track_dataset/test_filtered_track_dataset.py | 2 +- .../plugin_datastore/test_polars_track_store.py | 4 +++- .../plugin_datastore/test_python_track_store.py | 4 +++- .../plugin_datastore/test_track_store.py | 2 +- 12 files changed, 37 insertions(+), 24 deletions(-) diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index de58269e1..8b3ffb101 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -10,6 +10,7 @@ TrackFileRepository, TrackRepository, ) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class AllTrackIdsProvider(TrackIdProvider): @@ -150,7 +151,7 @@ def __call__(self, track_ids: Iterable[TrackId]) -> None: track_ids (Iterable[TrackId]): ids of tracks to be removed. """ try: - self._track_repository.remove_multiple(set(track_ids)) + self._track_repository.remove_multiple(PythonTrackIdSet(track_ids)) except RemoveMultipleTracksError as cause: logger().info(cause) diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 75eb1ac0d..6a9994b52 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -166,7 +166,7 @@ def remove(self, track_id: TrackId) -> "TrackDataset": raise NotImplementedError @abstractmethod - def remove_multiple(self, track_ids: set[TrackId]) -> "TrackDataset": + def remove_multiple(self, track_ids: TrackIdSet) -> "TrackDataset": raise NotImplementedError @abstractmethod diff --git a/OTAnalytics/domain/track_repository.py b/OTAnalytics/domain/track_repository.py index 8422f7f15..94e6d6b4c 100644 --- a/OTAnalytics/domain/track_repository.py +++ b/OTAnalytics/domain/track_repository.py @@ -198,8 +198,8 @@ def remove(self, track_id: TrackId) -> None: # application layer self.observers.notify(TrackRepositoryEvent.create_removed([track_id])) - def remove_multiple(self, track_ids: set[TrackId]) -> None: - not_existing_tracks = track_ids - set(self._dataset.track_ids) + def remove_multiple(self, track_ids: TrackIdSet) -> None: + not_existing_tracks = track_ids.difference(self._dataset.track_ids) if not_existing_tracks: logger().warning( f"Trying to remove {len(not_existing_tracks)} " diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 8e89bbfa0..d41e39b2c 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -425,8 +425,8 @@ def remove(self, track_id: TrackId) -> "PolarsTrackDataset": self.calculator, ) - def remove_multiple(self, track_ids: set[TrackId]) -> "PolarsTrackDataset": - track_id_strings = [unpack(track_id) for track_id in track_ids] + def remove_multiple(self, track_ids: TrackIdSet) -> "PolarsTrackDataset": + track_id_strings = self.__to_polars_series(track_ids) filtered_data = self._dataset.filter( ~pl.col(LEVEL_TRACK_ID).is_in(track_id_strings) ) @@ -437,6 +437,11 @@ def remove_multiple(self, track_ids: set[TrackId]) -> "PolarsTrackDataset": self.calculator, ) + def __to_polars_series(self, track_ids: TrackIdSet) -> pl.Series | list: + if isinstance(track_ids, PolarsTrackIdSet): + return track_ids._series + return [unpack(track_id) for track_id in track_ids] + def as_list(self) -> list[Track]: return list(self.as_generator()) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 7b2280e93..9cb84ea81 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -513,12 +513,14 @@ def get_for(self, id: TrackId) -> Optional[Track]: def remove(self, track_id: TrackId) -> "PythonTrackDataset": new_tracks = self._tracks.copy() del new_tracks[track_id] - updated_geometry_datasets = self._remove_from_geometry_datasets({track_id}) + updated_geometry_datasets = self._remove_from_geometry_datasets( + PythonTrackIdSet({track_id}) + ) return PythonTrackDataset( self.track_geometry_factory, new_tracks, updated_geometry_datasets ) - def remove_multiple(self, track_ids: set[TrackId]) -> "PythonTrackDataset": + def remove_multiple(self, track_ids: TrackIdSet) -> "PythonTrackDataset": new_tracks = self._tracks.copy() for track_id in track_ids: del new_tracks[track_id] @@ -528,7 +530,7 @@ def remove_multiple(self, track_ids: set[TrackId]) -> "PythonTrackDataset": ) def _remove_from_geometry_datasets( - self, track_ids: Iterable[TrackId] + self, track_ids: TrackIdSet ) -> dict[RelativeOffsetCoordinate, TrackGeometryDataset]: updated = {} for offset, geometry_dataset in self._geometry_datasets.items(): @@ -720,7 +722,7 @@ def revert_cuts_for(self, original_track_ids: TrackIdSet) -> "PythonTrackDataset for original_id, track_part in tracks_to_revert.items(): reverted_tracks.append(self.__revert_cut_for(original_id, track_part)) - result = self.remove_multiple(track_ids_to_remove) + result = self.remove_multiple(PythonTrackIdSet(track_ids_to_remove)) return result.add_all(reverted_tracks) def _create_original_track( @@ -833,7 +835,7 @@ def _get_dataset_with_classes(self, classes: list[str]) -> PythonTrackDataset: tracks_to_remove.append(_track.id) updated_geometry_datasets = self._other._remove_from_geometry_datasets( - tracks_to_remove + PythonTrackIdSet(tracks_to_remove) ) return PythonTrackDataset( self._other.track_geometry_factory, @@ -853,7 +855,7 @@ def add_all(self, other: Iterable[Track]) -> "TrackDataset": def remove(self, track_id: TrackId) -> "TrackDataset": return self.wrap(self._other.remove(track_id)) - def remove_multiple(self, track_ids: set[TrackId]) -> "TrackDataset": + def remove_multiple(self, track_ids: TrackIdSet) -> "TrackDataset": return self.wrap(self._other.remove_multiple(track_ids)) def clear(self) -> "TrackDataset": diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 9be5a3a3e..d19546ef0 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -403,8 +403,7 @@ def remove(self, track_id: TrackId) -> "PandasTrackDataset": remaining_tracks, self.track_geometry_factory, updated_geometry_datasets ) - def remove_multiple(self, track_ids: set[TrackId]) -> "PandasTrackDataset": - # TODO change to TrackIdSet + def remove_multiple(self, track_ids: TrackIdSet) -> "PandasTrackDataset": track_ids_primitive = [unpack(track_id) for track_id in track_ids] remaining_tracks = self._dataset.drop(track_ids_primitive, errors="ignore") updated_geometry_datasets = self._remove_from_geometry_dataset( @@ -719,7 +718,7 @@ def add_all(self, other: Iterable[Track]) -> PandasTrackDataset: def remove(self, track_id: TrackId) -> PandasTrackDataset: return self.wrap(self._other.remove(track_id)) - def remove_multiple(self, track_ids: set[TrackId]) -> PandasTrackDataset: + def remove_multiple(self, track_ids: TrackIdSet) -> PandasTrackDataset: return self.wrap(self._other.remove_multiple(track_ids)) def clear(self) -> PandasTrackDataset: diff --git a/tests/OTAnalytics/application/use_cases/test_track_repository.py b/tests/OTAnalytics/application/use_cases/test_track_repository.py index cdd11fbab..2391df732 100644 --- a/tests/OTAnalytics/application/use_cases/test_track_repository.py +++ b/tests/OTAnalytics/application/use_cases/test_track_repository.py @@ -145,10 +145,10 @@ def test_clear_all_tracks(self, track_repository: Mock) -> None: class TestRemoveTracks: def test_remove(self, track_repository: Mock) -> None: - remove_ids = [TrackId("1"), TrackId("2")] + remove_ids = PythonTrackIdSet([TrackId("1"), TrackId("2")]) remove_tracks = RemoveTracks(track_repository) remove_tracks(remove_ids) - track_repository.remove_multiple.assert_called_once_with(set(remove_ids)) + track_repository.remove_multiple.assert_called_once_with(remove_ids) class TestGetTracksFromIds: diff --git a/tests/OTAnalytics/domain/test_track_repository.py b/tests/OTAnalytics/domain/test_track_repository.py index 003c70988..cd6232938 100644 --- a/tests/OTAnalytics/domain/test_track_repository.py +++ b/tests/OTAnalytics/domain/test_track_repository.py @@ -137,7 +137,7 @@ def test_remove(self, track_1: Track, track_2: Track) -> None: ] def test_remove_multiple(self, track_1: Track, track_2: Track) -> None: - tracks_to_remove = frozenset([track_1.id, track_2.id]) + tracks_to_remove = PythonTrackIdSet([track_1.id, track_2.id]) updated_dataset = Mock(spec=TrackDataset) type(updated_dataset).track_ids = PropertyMock(return_value=tracks_to_remove) dataset = Mock(spec=TrackDataset) @@ -148,8 +148,10 @@ def test_remove_multiple(self, track_1: Track, track_2: Track) -> None: observer = Mock(spec=TrackListObserver) repository.register_tracks_observer(observer) - repository.remove_multiple({track_1.id, track_2.id}) - dataset.remove_multiple.assert_called_once_with({track_1.id, track_2.id}) + repository.remove_multiple(PythonTrackIdSet({track_1.id, track_2.id})) + dataset.remove_multiple.assert_called_once_with( + PythonTrackIdSet({track_1.id, track_2.id}) + ) assert observer.notify_tracks.call_args_list == [ call(TrackRepositoryEvent.create_removed(tracks_to_remove)) ] diff --git a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py index 78d4253e9..f87c06fd5 100644 --- a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py +++ b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py @@ -369,7 +369,7 @@ def test_remove_multiple( ) for filtered_dataset in filtered_datasets.values(): result = filtered_dataset.remove_multiple( - {cargo_bike_track.id, bicycle_track.id} + PythonTrackIdSet({cargo_bike_track.id, bicycle_track.id}) ) assert isinstance(result, type(filtered_dataset)) assert result.include_classes == filtered_dataset.include_classes diff --git a/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py index 9b98bc30f..df0204187 100644 --- a/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py @@ -264,7 +264,9 @@ def test_remove_multiple( track_geometry_factory, ) - result = dataset.remove_multiple({TrackId("second"), TrackId("fourth")}) + result = dataset.remove_multiple( + PolarsTrackIdSet({TrackId("second"), TrackId("fourth")}) + ) assert len(result) == 2 track_ids = {track_id.id for track_id in result.track_ids} diff --git a/tests/OTAnalytics/plugin_datastore/test_python_track_store.py b/tests/OTAnalytics/plugin_datastore/test_python_track_store.py index 52044e5c0..4b406aeef 100644 --- a/tests/OTAnalytics/plugin_datastore/test_python_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_python_track_store.py @@ -463,7 +463,9 @@ def test_remove_multiple(self, car_track: Track, pedestrian_track: Track) -> Non assert list(dataset) == [car_track, pedestrian_track] result = cast( PythonTrackDataset, - dataset.remove_multiple({car_track.id, pedestrian_track.id}), + dataset.remove_multiple( + PythonTrackIdSet({car_track.id, pedestrian_track.id}) + ), ) assert list(result) == [] assert result._geometry_datasets == { diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 4f47bcefc..609ac99a5 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -358,7 +358,7 @@ def test_remove_multiple( all_track_ids = PythonTrackIdSet( [first_track.id, second_track.id, third_track.id] ) - track_ids_to_remove = {first_track.id, second_track.id} + track_ids_to_remove = PythonTrackIdSet({first_track.id, second_track.id}) tracks_df = _convert_tracks([first_track, second_track, third_track]) geometry_dataset, updated_geometry_dataset = create_mock_geometry_dataset() dataset = PandasTrackDataset.from_dataframe( From c98d540b18d998abe3cb3934ca15c32d12a615f8 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 2 Sep 2025 21:42:32 +0200 Subject: [PATCH 054/128] Use TrackIdSet methods to speed up track statistics --- .../use_cases/highlight_intersections.py | 6 +++--- .../application/use_cases/track_statistics.py | 6 ++---- .../plugin_datastore/polars_track_id_set.py | 15 ++++++++++++--- OTAnalytics/plugin_filter/pandas_track_id.py | 12 ++++++------ 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/OTAnalytics/application/use_cases/highlight_intersections.py b/OTAnalytics/application/use_cases/highlight_intersections.py index d1e686a47..36584eb14 100644 --- a/OTAnalytics/application/use_cases/highlight_intersections.py +++ b/OTAnalytics/application/use_cases/highlight_intersections.py @@ -196,9 +196,9 @@ def __init__( self._track_repository = track_repository def get_ids(self) -> TrackIdSet: - all_track_ids = {track.id for track in self._track_repository.get_all()} - assigned_tracks = set(self._track_id_provider.get_ids()) - return PythonTrackIdSet(all_track_ids - assigned_tracks) + all_track_ids = self._track_repository.get_all_ids() + assigned_tracks = self._track_id_provider.get_ids() + return all_track_ids.difference(assigned_tracks) class TracksAssignedToSelectedFlows(TrackIdProvider): diff --git a/OTAnalytics/application/use_cases/track_statistics.py b/OTAnalytics/application/use_cases/track_statistics.py index befd87f75..3683d3511 100644 --- a/OTAnalytics/application/use_cases/track_statistics.py +++ b/OTAnalytics/application/use_cases/track_statistics.py @@ -89,10 +89,8 @@ def __init__( self._get_all_enter_section_events = get_all_enter_section_events def get_statistics(self) -> TrackStatistics: - ids_all = set(self._all_tracks.get_ids()) - ids_inside_cutting_sections = set( - self._track_ids_inside_cutting_sections.get_ids() - ) + ids_all = self._all_tracks.get_ids() + ids_inside_cutting_sections = self._track_ids_inside_cutting_sections.get_ids() track_count_inside = len(ids_inside_cutting_sections) track_count = len(ids_all) diff --git a/OTAnalytics/plugin_datastore/polars_track_id_set.py b/OTAnalytics/plugin_datastore/polars_track_id_set.py index ecbf0a06f..bf762d865 100644 --- a/OTAnalytics/plugin_datastore/polars_track_id_set.py +++ b/OTAnalytics/plugin_datastore/polars_track_id_set.py @@ -1,4 +1,4 @@ -from typing import Iterable, Iterator, Any +from typing import Any, Iterable, Iterator import polars as pl @@ -61,5 +61,14 @@ def union(self, other: "TrackIdSet") -> "TrackIdSet": return PolarsTrackIdSet(combined_series) def difference(self, other: "TrackIdSet") -> "TrackIdSet": - # TODO implement - raise NotImplementedError \ No newline at end of file + if isinstance(other, PolarsTrackIdSet): + # Efficient difference using Polars operations + difference_series = self._series.filter(~self._series.is_in(other._series)) + return PolarsTrackIdSet(difference_series) + else: + # Convert other to set of strings for difference + other_strings = {unpack(track_id) for track_id in other} + difference_series = self._series.filter( + ~self._series.is_in(list(other_strings)) + ) + return PolarsTrackIdSet(difference_series) diff --git a/OTAnalytics/plugin_filter/pandas_track_id.py b/OTAnalytics/plugin_filter/pandas_track_id.py index 4b4bf8ef0..9034bc174 100644 --- a/OTAnalytics/plugin_filter/pandas_track_id.py +++ b/OTAnalytics/plugin_filter/pandas_track_id.py @@ -1,7 +1,9 @@ -from OTAnalytics.domain.track import TRACK_ID, TrackId +import polars as pl + +from OTAnalytics.domain.track import TRACK_ID from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackIdSet from OTAnalytics.domain.track_id_provider import TrackIdProvider -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet +from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSet from OTAnalytics.plugin_datastore.track_store import PandasDataFrameProvider @@ -20,7 +22,5 @@ def get_ids(self) -> TrackIdSet: f"{TRACK_ID}" "must be in the index of DataFrame for retrieving all track ids." ) - - return PythonTrackIdSet( - [TrackId(id) for id in data.index.get_level_values(TRACK_ID)] - ) + series = data.index.get_level_values(TRACK_ID).unique() + return PolarsTrackIdSet(pl.from_pandas(series).cast(pl.Utf8)) From aff4371acc44dac6102df13084a78b229de3209f Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 17 Sep 2025 17:35:10 +0200 Subject: [PATCH 055/128] Remove unused method --- .../plugin_datastore/polars_track_store.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index d41e39b2c..25cc9ee4e 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -638,24 +638,6 @@ def cut_with_section( PolarsTrackIdSet(), ) - def _add_cut_track_part( - self, - cut_tracks_data: list, - detections: list[dict], - original_track_id_str: str, - original_id: str, - part_index: int, - ) -> None: - """Add a cut track part to the cut tracks data list.""" - new_track_id = f"{original_track_id_str}_{part_index}" - - for detection in detections: - # Create a copy of detection with new track ID - detection_copy = detection.copy() - detection_copy[track.TRACK_ID] = new_track_id - detection_copy[track.ORIGINAL_TRACK_ID] = original_id - cut_tracks_data.append(detection_copy) - def filter_by_min_detection_length(self, length: int) -> TrackDataset: if self._dataset.is_empty(): return self From 5cee394300b89f8bab2a28ba47fd359b0cb2bf23 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 15:40:36 +0200 Subject: [PATCH 056/128] Fix broken test --- .../use_cases/test_highlight_intersections.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py index a00b547b5..c19c8ea22 100644 --- a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py +++ b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py @@ -35,7 +35,7 @@ from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Detection, Track, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackIdSet from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @@ -280,18 +280,22 @@ def test_get_merged_ids( class TestTracksNotIntersectingSelection: def test_get_ids(self, track_1: Mock, track_2: Mock) -> None: track_repository = Mock(spec=TrackRepository) - track_repository.get_all.return_value = [track_1, track_2] + track_repository.get_all_ids.return_value = PythonTrackIdSet( + [track_1.id, track_2.id] + ) tracks_intersecting_sections = Mock(spec=TrackIdProvider) - tracks_intersecting_sections.get_ids.return_value = {track_1.id} + tracks_intersecting_sections.get_ids.return_value = PythonTrackIdSet( + [track_1.id] + ) tracks_not_intersecting_sections = TracksNotIntersectingSelection( tracks_intersecting_sections, track_repository ) - track_ids = list(tracks_not_intersecting_sections.get_ids()) + track_ids = tracks_not_intersecting_sections.get_ids() - assert track_ids == [track_2.id] - track_repository.get_all.assert_called_once() + assert track_ids == PythonTrackIdSet([track_2.id]) + track_repository.get_all_ids.assert_called_once() tracks_intersecting_sections.get_ids.assert_called_once() def test_no_selection_returns_all_tracks( @@ -300,18 +304,20 @@ def test_no_selection_returns_all_tracks( track_1 = Mock(spec=Track) track_2 = Mock(spec=Track) track_repository = Mock(spec=TrackRepository) - track_repository.get_all.return_value = [track_1, track_2] + track_repository.get_all_ids.return_value = PythonTrackIdSet( + [track_1.id, track_2.id] + ) tracks_intersecting_sections = Mock(spec=TrackIdProvider) - tracks_intersecting_sections.get_ids.return_value = {} + tracks_intersecting_sections.get_ids.return_value = EmptyTrackIdSet() tracks_not_intersecting_sections = TracksNotIntersectingSelection( tracks_intersecting_sections, track_repository ) track_ids = tracks_not_intersecting_sections.get_ids() - assert set(track_ids) == {track_1.id, track_2.id} - track_repository.get_all.assert_called_once() + assert track_ids == PythonTrackIdSet([track_1.id, track_2.id]) + track_repository.get_all_ids.assert_called_once() tracks_intersecting_sections.get_ids.assert_called_once() From e683f4d0c13e2dde7521aaf0cb95ec7f0f011284 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 15:41:36 +0200 Subject: [PATCH 057/128] Encapsulate resulting intersection points --- .../use_cases/create_intersection_events.py | 4 +- .../domain/track_dataset/track_dataset.py | 101 ++++++++++++++++++ .../test_create_intersection_events.py | 23 ++-- 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 308369c03..5160cd83c 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -56,7 +56,9 @@ def __do_intersect( offset: RelativeOffsetCoordinate, event_builder: EventBuilder, ) -> list[Event]: - intersection_result = track_dataset.intersection_points(sections, offset) + # TODO Create an EventDataset and internally use the PolarsDataFrame + # The for loop kills performance + intersection_result = track_dataset.wrap_intersection_points(sections, offset) events: list[Event] = [] for track_id, intersection_points in intersection_result.items(): diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 6a9994b52..4f4e5eba0 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -212,6 +212,22 @@ def intersection_points( """ raise NotImplementedError + def wrap_intersection_points( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> "IntersectionPointsDataset": + """ + Return the intersection points wrapped in an IntersectionPointsDataset. + + Args: + sections (list[Section]): the sections to intersect with. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. + + Returns: + IntersectionPointsDataset: the intersection points dataset. + """ + intersection_data = self.intersection_points(sections, offset) + return IntersectionPointsDataset(intersection_data) + @abstractmethod def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate @@ -416,6 +432,21 @@ def intersection_points( """ raise NotImplementedError + def wrap_intersection_points( + self, sections: list[Section] + ) -> "IntersectionPointsDataset": + """ + Return the intersection points wrapped in an IntersectionPointsDataset. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + IntersectionPointsDataset: the intersection points dataset. + """ + intersection_data = self.intersection_points(sections) + return IntersectionPointsDataset(intersection_data) + @abstractmethod def contained_by_sections( self, sections: list[Section] @@ -436,6 +467,76 @@ def __eq__(self, other: Any) -> bool: raise NotImplementedError +class IntersectionPointsDataset: + """Dataset containing intersection points between tracks and sections.""" + + def __init__(self, data: dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]): + """Initialize the intersection points dataset. + + Args: + data: Dictionary mapping track IDs to lists of intersection points + with section IDs. + """ + self._data = data + + def items( + self, + ) -> Iterator[tuple[TrackId, list[tuple[SectionId, IntersectionPoint]]]]: + """Iterate over track IDs and their intersection points. + + Returns: + Iterator yielding (track_id, intersection_points) tuples. + """ + return iter(self._data.items()) + + def keys(self) -> Iterator[TrackId]: + """Get all track IDs that have intersection points. + + Returns: + Iterator over track IDs. + """ + return iter(self._data.keys()) + + def get(self, track_id: TrackId) -> list[tuple[SectionId, IntersectionPoint]]: + """Get intersection points for a specific track ID. + + Args: + track_id: The track ID to get intersection points for. + + Returns: + List of (section_id, intersection_point) tuples for the track. + """ + return self._data.get(track_id, []) + + @property + def empty(self) -> bool: + """Check if the dataset is empty. + + Returns: + True if no intersection points exist, False otherwise. + """ + return len(self._data) == 0 + + def __len__(self) -> int: + """Get the number of tracks with intersection points. + + Returns: + Number of tracks that have intersection points. + """ + return len(self._data) + + def __contains__(self, track_id: TrackId) -> bool: + """Check if a track ID has intersection points. + + Args: + track_id: The track ID to check. + + Returns: + True if the track has intersection points, False otherwise. + """ + return track_id in self._data + + TRACK_GEOMETRY_FACTORY = Callable[ [TrackDataset, RelativeOffsetCoordinate], TrackGeometryDataset ] diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index 24adcf561..2edf4f64a 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -25,6 +25,7 @@ from OTAnalytics.domain.track import Detection, Track from OTAnalytics.domain.track_dataset.track_dataset import ( IntersectionPoint, + IntersectionPointsDataset, TrackDataset, ) from OTAnalytics.domain.types import EventType @@ -129,7 +130,7 @@ def assert_valid(self, event_results: list, event_builder: Mock) -> None: self._assert_valid(event_results, event_builder) def _assert_valid(self, event_results: list, event_builder: Mock) -> None: - self.track_dataset.intersection_points.assert_called_once_with( + self.track_dataset.wrap_intersection_points.assert_called_once_with( [self.section], self.section.get_offset(EventType.SECTION_ENTER) ) if self.expected_event_coords: @@ -248,7 +249,9 @@ def test_case_track_line_section(track: Track) -> _TestCase: prev_detection = track.detections[ip.lower_index] detection = track.detections[ip.upper_index] track_dataset = Mock(spec=TrackDataset) - track_dataset.intersection_points.return_value = {track.id: [(section.id, ip)]} + track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset( + {track.id: [(section.id, ip)]} + ) track_dataset.get_for.return_value = track expected_event_coord = _ExpectedEventCoord.from_detection( detection, @@ -294,12 +297,14 @@ def test_case_closed_track_line_section( ) -> _TestCase: section = create_section([(0, 1.5), (3, 1.5)], SectionType.LINE) track_dataset = Mock(spec=TrackDataset) - track_dataset.intersection_points.return_value = { - closed_track.id: [ - (section.id, IntersectionPoint(upper_index=2, relative_position=0)), - (section.id, IntersectionPoint(upper_index=4, relative_position=0)), - ] - } + track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset( + { + closed_track.id: [ + (section.id, IntersectionPoint(upper_index=2, relative_position=0)), + (section.id, IntersectionPoint(upper_index=4, relative_position=0)), + ] + } + ) track_dataset.get_for.return_value = closed_track expected_event_coords = [ _ExpectedEventCoord(2, 2.0, 2.0), @@ -319,7 +324,7 @@ def test_case_closed_track_line_section( def test_case_line_section_no_intersection(track: Track) -> _TestCase: section = create_section([(0, 0), (10, 0)], SectionType.LINE) track_dataset = Mock(spec=TrackDataset) - track_dataset.intersection_points.return_value = {} + track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset({}) track_dataset.get_for.return_value = track return _TestCase(track, track_dataset, section, [], []) From f69e7278f7f2a67f6fc2b813f16a95dca35f1150 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 16:02:00 +0200 Subject: [PATCH 058/128] Move calculation of events into domain class to improve performance with other implementations --- OTAnalytics/application/analysis/intersect.py | 4 - .../use_cases/create_intersection_events.py | 50 +--------- .../domain/track_dataset/track_dataset.py | 96 +++++++++++++++---- .../test_create_intersection_events.py | 9 +- 4 files changed, 89 insertions(+), 70 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index 0d98e7b2e..c88b0bd3d 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -9,10 +9,6 @@ from OTAnalytics.domain.types import EventType -class IntersectionError(Exception): - pass - - class RunIntersect(ABC): """ Interface defining the use case to intersect the given tracks with the given diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 5160cd83c..0d9b6bd47 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -2,7 +2,6 @@ from typing import Callable, Iterable, Protocol from OTAnalytics.application.analysis.intersect import ( - IntersectionError, RunIntersect, group_sections_by_offset, ) @@ -15,7 +14,10 @@ ) from OTAnalytics.domain.intersect import Intersector, IntersectParallelizationStrategy from OTAnalytics.domain.section import Area, LineSection, Section -from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset +from OTAnalytics.domain.track_dataset.track_dataset import ( + IntersectionError, + TrackDataset, +) from OTAnalytics.domain.types import EventType @@ -56,50 +58,8 @@ def __do_intersect( offset: RelativeOffsetCoordinate, event_builder: EventBuilder, ) -> list[Event]: - # TODO Create an EventDataset and internally use the PolarsDataFrame - # The for loop kills performance intersection_result = track_dataset.wrap_intersection_points(sections, offset) - - events: list[Event] = [] - for track_id, intersection_points in intersection_result.items(): - if not (track := track_dataset.get_for(track_id)): - raise IntersectionError( - "Track not found. Unable to create intersection event " - f"for track {track_id}." - ) - event_builder.add_road_user_type(track.classification) - for section_id, intersection_point in intersection_points: - event_builder.add_section_id(section_id) - detection = track.get_detection(intersection_point.upper_index) - previous_detection = track.get_detection(intersection_point.lower_index) - current_coord = detection.get_coordinate(offset) - prev_coord = previous_detection.get_coordinate(offset) - direction_vector = self._calculate_direction_vector( - prev_coord.x, - prev_coord.y, - current_coord.x, - current_coord.y, - ) - interpolated_occurrence = self._get_interpolated_occurrence( - previous=previous_detection.occurrence, - current=detection.occurrence, - relative_position=intersection_point.relative_position, - ) - interpolated_event_coordinate = self._get_interpolated_event_coordinate( - previous=prev_coord, - current=current_coord, - relative_position=intersection_point.relative_position, - ) - event_builder.add_event_type(EventType.SECTION_ENTER) - event_builder.add_direction_vector(direction_vector) - event_builder.add_event_coordinate(current_coord.x, current_coord.y) - event_builder.add_interpolated_occurrence(interpolated_occurrence) - event_builder.add_interpolated_event_coordinate( - interpolated_event_coordinate.x, interpolated_event_coordinate.y - ) - events.append(event_builder.create_event(detection)) - - return events + return intersection_result.create_events(offset, event_builder) def _get_interpolated_occurrence( self, previous: datetime, current: datetime, relative_position: float diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 4f4e5eba0..8290b4c6f 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -3,9 +3,15 @@ from datetime import datetime from typing import Any, Callable, Iterable, Iterator, Optional, Sequence -from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.event import Event, EventBuilder, SectionEventBuilder +from OTAnalytics.domain.geometry import ( + Coordinate, + RelativeOffsetCoordinate, + calculate_direction_vector, +) from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.types import EventType START_X: str = "start_x" START_Y: str = "start_y" @@ -226,7 +232,7 @@ def wrap_intersection_points( IntersectionPointsDataset: the intersection points dataset. """ intersection_data = self.intersection_points(sections, offset) - return IntersectionPointsDataset(intersection_data) + return IntersectionPointsDataset(intersection_data, self) @abstractmethod def contained_by_sections( @@ -432,21 +438,6 @@ def intersection_points( """ raise NotImplementedError - def wrap_intersection_points( - self, sections: list[Section] - ) -> "IntersectionPointsDataset": - """ - Return the intersection points wrapped in an IntersectionPointsDataset. - - Args: - sections (list[Section]): the sections to intersect with. - - Returns: - IntersectionPointsDataset: the intersection points dataset. - """ - intersection_data = self.intersection_points(sections) - return IntersectionPointsDataset(intersection_data) - @abstractmethod def contained_by_sections( self, sections: list[Section] @@ -470,7 +461,11 @@ def __eq__(self, other: Any) -> bool: class IntersectionPointsDataset: """Dataset containing intersection points between tracks and sections.""" - def __init__(self, data: dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]): + def __init__( + self, + data: dict[TrackId, list[tuple[SectionId, IntersectionPoint]]], + track_dataset: TrackDataset, + ) -> None: """Initialize the intersection points dataset. Args: @@ -478,6 +473,7 @@ def __init__(self, data: dict[TrackId, list[tuple[SectionId, IntersectionPoint]] with section IDs. """ self._data = data + self._track_dataset = track_dataset def items( self, @@ -536,7 +532,71 @@ def __contains__(self, track_id: TrackId) -> bool: """ return track_id in self._data + def create_events( + self, + offset: RelativeOffsetCoordinate, + event_builder: EventBuilder = SectionEventBuilder(), + ) -> list[Event]: + # TODO Create an EventDataset and internally use the PolarsDataFrame + # The for loop kills performance + events: list[Event] = [] + for track_id, intersection_points in self.items(): + if not (track := self._track_dataset.get_for(track_id)): + raise IntersectionError( + "Track not found. Unable to create intersection event " + f"for track {track_id}." + ) + event_builder.add_road_user_type(track.classification) + for section_id, intersection_point in intersection_points: + event_builder.add_section_id(section_id) + detection = track.get_detection(intersection_point.upper_index) + previous_detection = track.get_detection(intersection_point.lower_index) + current_coord = detection.get_coordinate(offset) + prev_coord = previous_detection.get_coordinate(offset) + direction_vector = calculate_direction_vector( + prev_coord.x, + prev_coord.y, + current_coord.x, + current_coord.y, + ) + interpolated_occurrence = self._get_interpolated_occurrence( + previous=previous_detection.occurrence, + current=detection.occurrence, + relative_position=intersection_point.relative_position, + ) + interpolated_event_coordinate = self._get_interpolated_event_coordinate( + previous=prev_coord, + current=current_coord, + relative_position=intersection_point.relative_position, + ) + event_builder.add_event_type(EventType.SECTION_ENTER) + event_builder.add_direction_vector(direction_vector) + event_builder.add_event_coordinate(current_coord.x, current_coord.y) + event_builder.add_interpolated_occurrence(interpolated_occurrence) + event_builder.add_interpolated_event_coordinate( + interpolated_event_coordinate.x, interpolated_event_coordinate.y + ) + events.append(event_builder.create_event(detection)) + + return events + + def _get_interpolated_occurrence( + self, previous: datetime, current: datetime, relative_position: float + ) -> datetime: + return previous + (current - previous) * relative_position + + def _get_interpolated_event_coordinate( + self, previous: Coordinate, current: Coordinate, relative_position: float + ) -> Coordinate: + interpolated_x = previous.x + relative_position * (current.x - previous.x) + interpolated_y = previous.y + relative_position * (current.y - previous.y) + return Coordinate(interpolated_x, interpolated_y) + TRACK_GEOMETRY_FACTORY = Callable[ [TrackDataset, RelativeOffsetCoordinate], TrackGeometryDataset ] + + +class IntersectionError(Exception): + pass diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index 2edf4f64a..e0762b3f6 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -250,7 +250,7 @@ def test_case_track_line_section(track: Track) -> _TestCase: detection = track.detections[ip.upper_index] track_dataset = Mock(spec=TrackDataset) track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset( - {track.id: [(section.id, ip)]} + {track.id: [(section.id, ip)]}, track_dataset ) track_dataset.get_for.return_value = track expected_event_coord = _ExpectedEventCoord.from_detection( @@ -303,7 +303,8 @@ def test_case_closed_track_line_section( (section.id, IntersectionPoint(upper_index=2, relative_position=0)), (section.id, IntersectionPoint(upper_index=4, relative_position=0)), ] - } + }, + track_dataset, ) track_dataset.get_for.return_value = closed_track expected_event_coords = [ @@ -324,7 +325,9 @@ def test_case_closed_track_line_section( def test_case_line_section_no_intersection(track: Track) -> _TestCase: section = create_section([(0, 0), (10, 0)], SectionType.LINE) track_dataset = Mock(spec=TrackDataset) - track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset({}) + track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset( + {}, track_dataset + ) track_dataset.get_for.return_value = track return _TestCase(track, track_dataset, section, [], []) From 64762de8878037576ab6f557606a1be6f682391f Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 16:29:22 +0200 Subject: [PATCH 059/128] Introduce EventDataset to encapsulate events. --- .../use_cases/create_intersection_events.py | 51 +++++++------ OTAnalytics/domain/event.py | 75 ++++++++++++++++++- OTAnalytics/domain/intersect.py | 4 +- .../domain/track_dataset/track_dataset.py | 13 ++-- .../test_create_intersection_events.py | 9 ++- 5 files changed, 118 insertions(+), 34 deletions(-) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 0d9b6bd47..31c6fe321 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -5,7 +5,12 @@ RunIntersect, group_sections_by_offset, ) -from OTAnalytics.domain.event import Event, EventBuilder, SectionEventBuilder +from OTAnalytics.domain.event import ( + Event, + EventBuilder, + EventDataset, + SectionEventBuilder, +) from OTAnalytics.domain.geometry import ( Coordinate, DirectionVector2D, @@ -40,16 +45,17 @@ def intersect( track_dataset: TrackDataset, sections: Iterable[Section], event_builder: EventBuilder, - ) -> list[Event]: + ) -> EventDataset: sections_grouped_by_offset = group_sections_by_offset( sections, EventType.SECTION_ENTER ) - events = [] + event_dataset = EventDataset() for offset, section_group in sections_grouped_by_offset.items(): - events.extend( - self.__do_intersect(track_dataset, section_group, offset, event_builder) + result = self.__do_intersect( + track_dataset, section_group, offset, event_builder ) - return events + event_dataset.extend(result) + return event_dataset def __do_intersect( self, @@ -57,7 +63,7 @@ def __do_intersect( sections: list[Section], offset: RelativeOffsetCoordinate, event_builder: EventBuilder, - ) -> list[Event]: + ) -> EventDataset: intersection_result = track_dataset.wrap_intersection_points(sections, offset) return intersection_result.create_events(offset, event_builder) @@ -88,16 +94,17 @@ def intersect( track_dataset: TrackDataset, sections: Iterable[Section], event_builder: EventBuilder, - ) -> list[Event]: + ) -> EventDataset: sections_grouped_by_offset = group_sections_by_offset( sections, EventType.SECTION_ENTER ) - events = [] + event_dataset = EventDataset() for offset, section_group in sections_grouped_by_offset.items(): - events.extend( - self.__do_intersect(track_dataset, section_group, offset, event_builder) + result = self.__do_intersect( + track_dataset, section_group, offset, event_builder ) - return events + event_dataset.extend(result) + return event_dataset def __do_intersect( self, @@ -105,7 +112,7 @@ def __do_intersect( sections: list[Section], offset: RelativeOffsetCoordinate, event_builder: EventBuilder, - ) -> list[Event]: + ) -> EventDataset: contained_by_sections_result = track_dataset.contained_by_sections( sections, offset ) @@ -184,7 +191,7 @@ def __do_intersect( events.append(event) section_currently_entered = entered - return events + return EventDataset(events) class RunCreateIntersectionEvents: @@ -202,20 +209,20 @@ def __init__( self._sections = sections self._event_builder = event_builder - def create(self) -> list[Event]: - events = [] + def create(self) -> EventDataset: + event_dataset = EventDataset() line_sections, area_sections = separate_sections(self._sections) - events.extend( + event_dataset.extend( self._intersect_line_section.intersect( self._track_dataset, line_sections, self._event_builder ) ) - events.extend( + event_dataset.extend( self._intersect_area_section.intersect( self._track_dataset, area_sections, self._event_builder ) ) - return events + return event_dataset class GetTracks(Protocol): @@ -244,7 +251,7 @@ def __call__(self, sections: Iterable[Section]) -> list[Event]: def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Event]: - events = [] + event_dataset = EventDataset() event_builder = SectionEventBuilder() create_intersection_events = RunCreateIntersectionEvents( @@ -254,8 +261,8 @@ def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Ev sections=sections, event_builder=event_builder, ) - events.extend(create_intersection_events.create()) - return events + event_dataset.extend(create_intersection_events.create()) + return event_dataset.get_events() def separate_sections( diff --git a/OTAnalytics/domain/event.py b/OTAnalytics/domain/event.py index 4acf753e2..2b3cc0713 100644 --- a/OTAnalytics/domain/event.py +++ b/OTAnalytics/domain/event.py @@ -4,7 +4,7 @@ from collections import defaultdict from dataclasses import dataclass from datetime import datetime -from typing import Callable, Iterable, Optional, Sequence +from typing import Callable, Iterable, Iterator, Optional, Sequence from OTAnalytics.domain.common import DataclassValidation from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate @@ -594,3 +594,76 @@ def after_filter(date: datetime) -> Callable[[Event], bool]: def before_filter(date: datetime) -> Callable[[Event], bool]: return lambda actual: actual.occurrence <= date + + +class EventDataset: + """A dataset wrapper for a collection of events. + + This class wraps a list of events and provides methods for combining + multiple datasets and accessing the events. + """ + + def __init__(self, events: list[Event] | None = None) -> None: + """Initialize the EventDataset. + + Args: + events: Optional list of events. If None, an empty list is created. + """ + self._events = events if events is not None else [] + + def __iter__(self) -> Iterator[Event]: + """Iterate over the events in the dataset.""" + return iter(self._events) + + def __len__(self) -> int: + """Return the number of events in the dataset.""" + return len(self._events) + + def __add__(self, other: "EventDataset") -> "EventDataset": + """Add two EventDatasets together to create a new combined dataset. + + Args: + other: Another EventDataset to combine with this one. + + Returns: + A new EventDataset containing events from both datasets. + """ + if not isinstance(other, EventDataset): + return NotImplemented + return EventDataset(self._events + other._events) + + def extend(self, other: "EventDataset") -> None: + """Extend this dataset with events from another dataset. + + Args: + other: Another EventDataset whose events will be added to this dataset. + """ + if isinstance(other, EventDataset): + self._events.extend(other._events) + else: + # Support extending with a list of events for compatibility + self._events.extend(other) + + def append(self, event: Event) -> None: + """Add a single event to the dataset. + + Args: + event: The event to add to the dataset. + """ + self._events.append(event) + + def get_events(self) -> list[Event]: + """Get the underlying list of events. + + Returns: + The list of events contained in this dataset. + """ + return self._events.copy() + + def is_empty(self) -> bool: + """Check if the dataset contains no events. + + Returns: + True if the dataset is empty, False otherwise. + """ + return len(self._events) == 0 diff --git a/OTAnalytics/domain/intersect.py b/OTAnalytics/domain/intersect.py index e62c6aa32..6f8a89bb4 100644 --- a/OTAnalytics/domain/intersect.py +++ b/OTAnalytics/domain/intersect.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Callable, Iterable, Sequence -from OTAnalytics.domain.event import Event, EventBuilder +from OTAnalytics.domain.event import Event, EventBuilder, EventDataset from OTAnalytics.domain.geometry import Coordinate, Line, Polygon from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset @@ -140,7 +140,7 @@ def intersect( track_dataset: TrackDataset, sections: Iterable[Section], event_builder: EventBuilder, - ) -> list[Event]: + ) -> EventDataset: """Intersect tracks with sections and generate events if they intersect. Args: diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 8290b4c6f..3e2d6d812 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -3,7 +3,12 @@ from datetime import datetime from typing import Any, Callable, Iterable, Iterator, Optional, Sequence -from OTAnalytics.domain.event import Event, EventBuilder, SectionEventBuilder +from OTAnalytics.domain.event import ( + Event, + EventBuilder, + EventDataset, + SectionEventBuilder, +) from OTAnalytics.domain.geometry import ( Coordinate, RelativeOffsetCoordinate, @@ -536,9 +541,7 @@ def create_events( self, offset: RelativeOffsetCoordinate, event_builder: EventBuilder = SectionEventBuilder(), - ) -> list[Event]: - # TODO Create an EventDataset and internally use the PolarsDataFrame - # The for loop kills performance + ) -> EventDataset: events: list[Event] = [] for track_id, intersection_points in self.items(): if not (track := self._track_dataset.get_for(track_id)): @@ -578,7 +581,7 @@ def create_events( ) events.append(event_builder.create_event(detection)) - return events + return EventDataset(events) def _get_interpolated_occurrence( self, previous: datetime, current: datetime, relative_position: float diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index e0762b3f6..4f62bb5d0 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -9,6 +9,7 @@ IntersectByIntersectionPoints, separate_sections, ) +from OTAnalytics.domain.event import EventDataset from OTAnalytics.domain.geometry import ( Coordinate, DirectionVector2D, @@ -71,7 +72,7 @@ def __init__( self.expected_event_coords = expected_event_coords self.direction_vectors = direction_vectors - def assert_valid(self, event_results: list, event_builder: Mock) -> None: + def assert_valid(self, event_results: EventDataset, event_builder: Mock) -> None: assert len(event_results) == len(self.expected_event_coords) if self.expected_event_coords: self._assert_has_calls(event_builder) @@ -125,11 +126,11 @@ def __init__( self.expected_interpolated_occurrences = expected_interpolated_occurrences self.expected_interpolated_coords = expected_interpolated_coords - def assert_valid(self, event_results: list, event_builder: Mock) -> None: + def assert_valid(self, event_results: EventDataset, event_builder: Mock) -> None: super().assert_valid(event_results, event_builder) self._assert_valid(event_results, event_builder) - def _assert_valid(self, event_results: list, event_builder: Mock) -> None: + def _assert_valid(self, event_results: EventDataset, event_builder: Mock) -> None: self.track_dataset.wrap_intersection_points.assert_called_once_with( [self.section], self.section.get_offset(EventType.SECTION_ENTER) ) @@ -158,7 +159,7 @@ def __init__( track, track_dataset, section, expected_event_coords, direction_vectors ) - def assert_valid(self, event_results: list, event_builder: Mock) -> None: + def assert_valid(self, event_results: EventDataset, event_builder: Mock) -> None: super().assert_valid(event_results, event_builder) self._assert_valid() From 53d874e84522a7744369b1cbf03244741e1e630f Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 20:33:13 +0200 Subject: [PATCH 060/128] Use EventDataset instead of bare list --- OTAnalytics/application/analysis/intersect.py | 4 +- .../application/analysis/traffic_counting.py | 28 +++--- OTAnalytics/application/eventlist.py | 15 +-- .../use_cases/create_intersection_events.py | 13 +-- OTAnalytics/domain/event.py | 5 + OTAnalytics/domain/intersect.py | 10 +- .../multiprocessing.py | 22 +++-- .../sequential.py | 12 +-- .../analysis/test_traffic_counting.py | 8 +- .../OTAnalytics/application/test_eventlist.py | 94 ++++++++++--------- .../test_multiprocessing.py | 14 +-- .../test_sequential.py | 4 +- 12 files changed, 120 insertions(+), 109 deletions(-) diff --git a/OTAnalytics/application/analysis/intersect.py b/OTAnalytics/application/analysis/intersect.py index c88b0bd3d..b74474058 100644 --- a/OTAnalytics/application/analysis/intersect.py +++ b/OTAnalytics/application/analysis/intersect.py @@ -2,7 +2,7 @@ from collections import defaultdict from typing import Iterable, Mapping -from OTAnalytics.domain.event import Event +from OTAnalytics.domain.event import EventDataset from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet @@ -16,7 +16,7 @@ class RunIntersect(ABC): """ @abstractmethod - def __call__(self, sections: Iterable[Section]) -> list[Event]: + def __call__(self, sections: Iterable[Section]) -> EventDataset: raise NotImplementedError diff --git a/OTAnalytics/application/analysis/traffic_counting.py b/OTAnalytics/application/analysis/traffic_counting.py index 51680f232..ba94ea81e 100644 --- a/OTAnalytics/application/analysis/traffic_counting.py +++ b/OTAnalytics/application/analysis/traffic_counting.py @@ -14,7 +14,7 @@ from OTAnalytics.application.export_formats.export_mode import ExportMode from OTAnalytics.application.use_cases.create_events import CreateEvents from OTAnalytics.application.use_cases.section_repository import GetSectionsById -from OTAnalytics.domain.event import Event, EventRepository +from OTAnalytics.domain.event import Event, EventDataset, EventRepository from OTAnalytics.domain.flow import Flow, FlowRepository from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.types import EventType @@ -681,9 +681,9 @@ class FilterBySectionEnterEvent(RoadUserAssignerDecorator): """Decorator to filters events by event type section-enter.""" def assign(self, events: Iterable[Event], flows: list[Flow]) -> RoadUserAssignments: - section_enter_events: list[Event] = [ - event for event in events if event.event_type == EventType.SECTION_ENTER - ] + section_enter_events = EventDataset( + [event for event in events if event.event_type == EventType.SECTION_ENTER] + ) return super().assign(section_enter_events, flows) @@ -741,7 +741,7 @@ def __group_flows_by_sections( def __group_events_by_road_user( self, events: Iterable[Event] - ) -> dict[tuple[str, str], list[Event]]: + ) -> dict[tuple[str, str], EventDataset]: """ Group events by road user. @@ -749,10 +749,10 @@ def __group_events_by_road_user( events (Iterable[Event]): events of a road user Returns: - dict[tuple[RoadUserId, RoadUserType], list[Event]]: events grouped by user + dict[tuple[RoadUserId, RoadUserType], EventDataset]: events grouped by user """ - events_by_road_user: dict[tuple[RoadUserId, RoadUserType], list[Event]] = ( - defaultdict(list) + events_by_road_user: dict[tuple[RoadUserId, RoadUserType], EventDataset] = ( + defaultdict(EventDataset) ) sorted_events = sorted( events, key=lambda _event: _event.interpolated_occurrence @@ -767,7 +767,7 @@ def __group_events_by_road_user( def __assign_user_to_flow( self, flows: dict[tuple[SectionId, SectionId], list[Flow]], - events_by_road_user: dict[tuple[str, str], list[Event]], + events_by_road_user: dict[tuple[str, str], EventDataset], ) -> RoadUserAssignments: """ Assign each user to flows based on the flow selection strategy. @@ -775,7 +775,7 @@ def __assign_user_to_flow( Args: flows (dict[tuple[SectionId, SectionId], list[Flow]]): flows by start and end section - events_by_road_user (dict[str, list[Event]]): events by road user + events_by_road_user (dict[str, EventDataset]): events by road user Returns: RoadUserAssignments: group of RoadUserAssignment objects @@ -793,7 +793,7 @@ def __assign_user_to_flow( def __create_candidates( self, flows: dict[tuple[SectionId, SectionId], list[Flow]], - events: list[Event], + events: EventDataset, ) -> list[FlowCandidate]: """ Create flow candidates to select one from in a later step. @@ -801,7 +801,7 @@ def __create_candidates( Args: flows (dict[tuple[SectionId, SectionId], list[Flow]]): flows by start and end section - events (list[Event]): events belonging to road user + events (EventDataset): events belonging to road user Returns: list[FlowCandidate]: the flow candidates pertaining to road user @@ -809,14 +809,14 @@ def __create_candidates( event_pairs = self.__create_event_pairs(events) return self.__create_candidate_flows(flows, event_pairs) - def __create_event_pairs(self, events: list[Event]) -> list[EventPair]: + def __create_event_pairs(self, events: EventDataset) -> list[EventPair]: """ Create event pairs. Requires and assumes events to be sorted by occurrence. Args: - events(list[Event]): events to create the event pairs with + events(EventDataset): events to create the event pairs with Returns: list[EventPair]: event pairs diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 128b2af2c..44683f8d7 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -5,6 +5,7 @@ FILE_NAME_PATTERN, HOSTNAME, Event, + EventDataset, ImproperFormattedFilename, ) from OTAnalytics.domain.geometry import ImageCoordinate, calculate_direction_vector @@ -52,7 +53,7 @@ class SceneEventListBuilder: """Create enter and leave scene events for track segments.""" def __init__(self) -> None: - self._events: list[Event] = [] + self._events = EventDataset() def add_enter_scene_events( self, segments: TrackSegmentDataset @@ -76,13 +77,13 @@ def add_leave_scene_events( segments.apply(self._create_leave_scene_event) return self - def build(self) -> list[Event]: - """Create the complete event list. + def build(self) -> EventDataset: + """Create the complete event dataset. Returns: - list[Event]: complete event list + EventDataset: complete event dataset """ - return self._events.copy() + return EventDataset(self._events.get_events()) def _create_enter_scene_event(self, value: dict) -> None: event = self.__create_event( @@ -144,14 +145,14 @@ def __create_event( class SceneActionDetector: """Detect when a road user enters or leaves the scene.""" - def detect(self, tracks: TrackDataset) -> list[Event]: + def detect(self, tracks: TrackDataset) -> EventDataset: """Detect all enter and leave scene events. Args: tracks (Iterable[Track]): the tracks under inspection Returns: - Iterable[Event]: the scene events + EventDataset: the scene events """ builder = SceneEventListBuilder() builder.add_enter_scene_events(tracks.get_first_segments()) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 31c6fe321..e31e000a8 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -5,12 +5,7 @@ RunIntersect, group_sections_by_offset, ) -from OTAnalytics.domain.event import ( - Event, - EventBuilder, - EventDataset, - SectionEventBuilder, -) +from OTAnalytics.domain.event import EventBuilder, EventDataset, SectionEventBuilder from OTAnalytics.domain.geometry import ( Coordinate, DirectionVector2D, @@ -238,7 +233,7 @@ def __init__( self._intersect_parallelizer = intersect_parallelizer self._get_tracks = get_tracks - def __call__(self, sections: Iterable[Section]) -> list[Event]: + def __call__(self, sections: Iterable[Section]) -> EventDataset: filtered_tracks = self._get_tracks.as_dataset() filtered_tracks.calculate_geometries_for( {_section.get_offset(EventType.SECTION_ENTER) for _section in sections} @@ -250,7 +245,7 @@ def __call__(self, sections: Iterable[Section]) -> list[Event]: return self._intersect_parallelizer.execute(_create_events, tasks) -def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Event]: +def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> EventDataset: event_dataset = EventDataset() event_builder = SectionEventBuilder() @@ -262,7 +257,7 @@ def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> list[Ev event_builder=event_builder, ) event_dataset.extend(create_intersection_events.create()) - return event_dataset.get_events() + return event_dataset def separate_sections( diff --git a/OTAnalytics/domain/event.py b/OTAnalytics/domain/event.py index 2b3cc0713..da6147968 100644 --- a/OTAnalytics/domain/event.py +++ b/OTAnalytics/domain/event.py @@ -632,6 +632,11 @@ def __add__(self, other: "EventDataset") -> "EventDataset": return NotImplemented return EventDataset(self._events + other._events) + def __eq__(self, other: object) -> bool: + if not isinstance(other, EventDataset): + return False + return self._events == other._events + def extend(self, other: "EventDataset") -> None: """Extend this dataset with events from another dataset. diff --git a/OTAnalytics/domain/intersect.py b/OTAnalytics/domain/intersect.py index 6f8a89bb4..20803099a 100644 --- a/OTAnalytics/domain/intersect.py +++ b/OTAnalytics/domain/intersect.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Callable, Iterable, Sequence -from OTAnalytics.domain.event import Event, EventBuilder, EventDataset +from OTAnalytics.domain.event import EventBuilder, EventDataset from OTAnalytics.domain.geometry import Coordinate, Line, Polygon from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset @@ -101,20 +101,20 @@ def num_processes(self) -> int: @abstractmethod def execute( self, - intersect: Callable[[TrackDataset, Iterable[Section]], Iterable[Event]], + intersect: Callable[[TrackDataset, Iterable[Section]], EventDataset], tasks: Sequence[tuple[TrackDataset, Iterable[Section]]], - ) -> list[Event]: + ) -> EventDataset: """Executes the intersection of tracks with sections with the implemented parallelization strategy. Args: - intersect (Callable[[TrackDataset, Iterable[Section]], Iterable[Event]]): + intersect (Callable[[TrackDataset, Iterable[Section]], EventDataset]): the function to be executed on an iterable of tracks and sections. tasks (Sequence[tuple[TrackDataset, Iterable[Section]]): the argument to intersect function. Returns: - Iterable[Event]: the generated events. + EventDataset: the generated events. """ raise NotImplementedError diff --git a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py index de425328e..6515799e5 100644 --- a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py +++ b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py @@ -1,10 +1,9 @@ -import itertools from multiprocessing import Pool from typing import Callable, Iterable, Sequence from OTAnalytics.application.config import DEFAULT_NUM_PROCESSES from OTAnalytics.application.logger import logger -from OTAnalytics.domain.event import Event +from OTAnalytics.domain.event import EventDataset from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset @@ -32,19 +31,24 @@ def set_num_processes(self, value: int) -> None: def execute( self, - intersect: Callable[[TrackDataset, Iterable[Section]], Iterable[Event]], + intersect: Callable[[TrackDataset, Iterable[Section]], EventDataset], tasks: Sequence[tuple[TrackDataset, Iterable[Section]]], - ) -> list[Event]: + ) -> EventDataset: logger().debug( f"Start intersection in parallel with {self._num_processes} processes." ) if self._num_processes > 1: with Pool(processes=self._num_processes) as pool: - events = pool.starmap(intersect, tasks) + event_datasets = pool.starmap(intersect, tasks) else: - events = [intersect(tracks, sections) for tracks, sections in tasks] + event_datasets = [intersect(tracks, sections) for tracks, sections in tasks] - return self._flatten_events(events) + return self._combine_event_datasets(event_datasets) - def _flatten_events(self, events: Iterable[Iterable[Event]]) -> list[Event]: - return list(itertools.chain.from_iterable(events)) + def _combine_event_datasets( + self, event_datasets: Iterable[EventDataset] + ) -> EventDataset: + combined = EventDataset() + for dataset in event_datasets: + combined.extend(dataset) + return combined diff --git a/OTAnalytics/plugin_intersect_parallelization/sequential.py b/OTAnalytics/plugin_intersect_parallelization/sequential.py index ecaadfb33..5afbbd809 100644 --- a/OTAnalytics/plugin_intersect_parallelization/sequential.py +++ b/OTAnalytics/plugin_intersect_parallelization/sequential.py @@ -1,6 +1,6 @@ from typing import Callable, Iterable, Sequence -from OTAnalytics.domain.event import Event +from OTAnalytics.domain.event import EventDataset from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset @@ -15,14 +15,14 @@ def num_processes(self) -> int: def execute( self, - intersect: Callable[[TrackDataset, Iterable[Section]], Iterable[Event]], + intersect: Callable[[TrackDataset, Iterable[Section]], EventDataset], tasks: Sequence[tuple[TrackDataset, Iterable[Section]]], - ) -> list[Event]: - events: list[Event] = [] + ) -> EventDataset: + event_dataset = EventDataset() for task in tasks: track_dataset, sections = task - events.extend(intersect(track_dataset, sections)) - return events + event_dataset.extend(intersect(track_dataset, sections)) + return event_dataset def set_num_processes(self, value: int) -> None: pass diff --git a/tests/OTAnalytics/application/analysis/test_traffic_counting.py b/tests/OTAnalytics/application/analysis/test_traffic_counting.py index ac45a3a9a..ec356ded2 100644 --- a/tests/OTAnalytics/application/analysis/test_traffic_counting.py +++ b/tests/OTAnalytics/application/analysis/test_traffic_counting.py @@ -49,7 +49,7 @@ from OTAnalytics.application.export_formats.export_mode import OVERWRITE from OTAnalytics.application.use_cases.create_events import CreateEvents from OTAnalytics.application.use_cases.section_repository import GetSectionsById -from OTAnalytics.domain.event import Event, EventRepository +from OTAnalytics.domain.event import Event, EventDataset, EventRepository from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate from OTAnalytics.domain.section import SectionId @@ -832,8 +832,10 @@ def test_assign_filters_by_section_enter_event(self) -> None: road_user_assigner = Mock(spec=RoadUserAssigner) decorator = FilterBySectionEnterEvent(road_user_assigner) - decorator.assign([enter_scene_event, section_enter_event], [flow]) - road_user_assigner.assign.assert_called_once_with([section_enter_event], [flow]) + decorator.assign(EventDataset([enter_scene_event, section_enter_event]), [flow]) + road_user_assigner.assign.assert_called_once_with( + EventDataset([section_enter_event]), [flow] + ) class TestSimpleRoadUserAssigner: diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index dcb73727f..313ae9d92 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -4,7 +4,7 @@ import pytest from OTAnalytics.application.eventlist import SceneActionDetector, SceneEventListBuilder -from OTAnalytics.domain.event import Event, EventType +from OTAnalytics.domain.event import Event, EventDataset, EventType from OTAnalytics.domain.geometry import ( Coordinate, ImageCoordinate, @@ -170,63 +170,67 @@ def test_detect(self, track: Track) -> None: last_segments.apply.assert_called_once() -def create_expected_events(track: Track) -> list[Event]: +def create_expected_events(track: Track) -> EventDataset: return create_expected_enter_scene_events( track ) + create_expected_leave_scene_events(track) -def create_expected_leave_scene_events(track: Track) -> list[Event]: +def create_expected_leave_scene_events(track: Track) -> EventDataset: occurrence = track.last_detection.occurrence event_coordinate = ImageCoordinate(track.last_detection.x, track.last_detection.y) - return [ - Event( - road_user_id=track.id.id, - road_user_type=track.classification, - hostname=HOSTNAME, - occurrence=occurrence, - frame_number=track.last_detection.frame, - section_id=None, - event_coordinate=event_coordinate, - event_type=EventType.LEAVE_SCENE, - direction_vector=calculate_direction_vector( - track.detections[-2].x, - track.detections[-2].y, - track.last_detection.x, - track.last_detection.y, + return EventDataset( + [ + Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=HOSTNAME, + occurrence=occurrence, + frame_number=track.last_detection.frame, + section_id=None, + event_coordinate=event_coordinate, + event_type=EventType.LEAVE_SCENE, + direction_vector=calculate_direction_vector( + track.detections[-2].x, + track.detections[-2].y, + track.last_detection.x, + track.last_detection.y, + ), + video_name=track.first_detection.video_name, + interpolated_occurrence=occurrence, + interpolated_event_coordinate=event_coordinate, ), - video_name=track.first_detection.video_name, - interpolated_occurrence=occurrence, - interpolated_event_coordinate=event_coordinate, - ), - ] + ] + ) -def create_expected_enter_scene_events(track: Track) -> list[Event]: +def create_expected_enter_scene_events(track: Track) -> EventDataset: event_coordinate = ImageCoordinate(track.first_detection.x, track.first_detection.y) occurrence = track.first_detection.occurrence - return [ - Event( - road_user_id=track.id.id, - road_user_type=track.classification, - hostname=HOSTNAME, - occurrence=occurrence, - frame_number=track.first_detection.frame, - section_id=None, - event_coordinate=event_coordinate, - event_type=EventType.ENTER_SCENE, - direction_vector=calculate_direction_vector( - track.first_detection.x, - track.first_detection.y, - track.detections[1].x, - track.detections[1].y, - ), - video_name=track.first_detection.video_name, - interpolated_occurrence=occurrence, - interpolated_event_coordinate=event_coordinate, - ) - ] + return EventDataset( + [ + Event( + road_user_id=track.id.id, + road_user_type=track.classification, + hostname=HOSTNAME, + occurrence=occurrence, + frame_number=track.first_detection.frame, + section_id=None, + event_coordinate=event_coordinate, + event_type=EventType.ENTER_SCENE, + direction_vector=calculate_direction_vector( + track.first_detection.x, + track.first_detection.y, + track.detections[1].x, + track.detections[1].y, + ), + video_name=track.first_detection.video_name, + interpolated_occurrence=occurrence, + interpolated_event_coordinate=event_coordinate, + ) + ] + ) def first_segment_of(track: Track) -> dict: diff --git a/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py b/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py index cb7fc7cd8..ff41f13fe 100644 --- a/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py +++ b/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py @@ -3,7 +3,7 @@ import pytest -from OTAnalytics.domain.event import Event +from OTAnalytics.domain.event import Event, EventDataset from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset from OTAnalytics.plugin_intersect_parallelization.multiprocessing import ( @@ -26,7 +26,7 @@ def test_execute(self, mock_pool_init: Mock) -> None: parallelizer = MultiprocessingIntersectParallelization() result = parallelizer.execute(cast(Callable, intersect), tasks) - assert result == [event_1, event_2] + assert result == EventDataset([event_1, event_2]) mock_pool_instance.starmap.assert_called_once_with(intersect, tasks) @patch("OTAnalytics.plugin_intersect_parallelization.multiprocessing.Pool") @@ -37,7 +37,7 @@ def test_execute_sequentially(self, mock_pool_init: Mock) -> None: mock_pool_instance = mock_pool_init.return_value.__enter__.return_value intersect = Mock() - intersect.side_effect = [[event_1, event_2]] + intersect.side_effect = [EventDataset([event_1, event_2])] track_dataset: TrackDataset = Mock(spec=TrackDataset) section = Mock() task: tuple[TrackDataset, Iterable[Section]] = (track_dataset, [section]) @@ -45,7 +45,7 @@ def test_execute_sequentially(self, mock_pool_init: Mock) -> None: parallelizer = MultiprocessingIntersectParallelization(num_processes=1) result = parallelizer.execute(cast(Callable, intersect), [task]) - assert result == [event_1, event_2] + assert result == EventDataset([event_1, event_2]) mock_pool_instance.starmap.assert_not_called() intersect.assert_called_once_with(track_dataset, [section]) @@ -53,10 +53,10 @@ def test_flatten_events(self) -> None: intersect = MultiprocessingIntersectParallelization() event_1 = Mock(spec=Event) event_2 = Mock(spec=Event) - events_to_flatten = [[event_1], [event_2]] + events_to_flatten = [EventDataset([event_1]), EventDataset([event_2])] - result = intersect._flatten_events(events_to_flatten) - assert result == [event_1, event_2] + result = intersect._combine_event_datasets(events_to_flatten) + assert result == EventDataset([event_1, event_2]) def test_set_num_processes(self) -> None: intersect = MultiprocessingIntersectParallelization(4) diff --git a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py index 0b80a4cb7..61374d447 100644 --- a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py +++ b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py @@ -1,7 +1,7 @@ from typing import Callable, cast from unittest.mock import Mock, call -from OTAnalytics.domain.event import Event +from OTAnalytics.domain.event import Event, EventDataset from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset from OTAnalytics.plugin_intersect_parallelization.sequential import SequentialIntersect @@ -25,7 +25,7 @@ def test_execute(self) -> None: sequential_intersect = SequentialIntersect() result = sequential_intersect.execute(cast(Callable, mock_intersect), tasks) - assert result == [event_1, event_2] + assert result == EventDataset([event_1, event_2]) assert mock_intersect.call_args_list == [ call(first_track_dataset, sections), call(second_track_dataset, sections), From 7c6f080b9f161c33fd6afe80e95d2a26cdf8201a Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 20:35:58 +0200 Subject: [PATCH 061/128] Remove unused code --- .../use_cases/create_intersection_events.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index e31e000a8..f098e2cd2 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -1,4 +1,3 @@ -from datetime import datetime from typing import Callable, Iterable, Protocol from OTAnalytics.application.analysis.intersect import ( @@ -7,7 +6,6 @@ ) from OTAnalytics.domain.event import EventBuilder, EventDataset, SectionEventBuilder from OTAnalytics.domain.geometry import ( - Coordinate, DirectionVector2D, RelativeOffsetCoordinate, calculate_direction_vector, @@ -62,18 +60,6 @@ def __do_intersect( intersection_result = track_dataset.wrap_intersection_points(sections, offset) return intersection_result.create_events(offset, event_builder) - def _get_interpolated_occurrence( - self, previous: datetime, current: datetime, relative_position: float - ) -> datetime: - return previous + (current - previous) * relative_position - - def _get_interpolated_event_coordinate( - self, previous: Coordinate, current: Coordinate, relative_position: float - ) -> Coordinate: - interpolated_x = previous.x + relative_position * (current.x - previous.x) - interpolated_y = previous.y + relative_position * (current.y - previous.y) - return Coordinate(interpolated_x, interpolated_y) - class IntersectAreaByTrackPoints(Intersector): def __init__( From 07cd29e3827d8c2c3d7ccc412ad3be74cc4765a2 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 20:49:01 +0200 Subject: [PATCH 062/128] Raise exception --- OTAnalytics/domain/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/domain/event.py b/OTAnalytics/domain/event.py index da6147968..b7fa831dc 100644 --- a/OTAnalytics/domain/event.py +++ b/OTAnalytics/domain/event.py @@ -629,7 +629,7 @@ def __add__(self, other: "EventDataset") -> "EventDataset": A new EventDataset containing events from both datasets. """ if not isinstance(other, EventDataset): - return NotImplemented + raise TypeError() return EventDataset(self._events + other._events) def __eq__(self, other: object) -> bool: From a4a501cbf56aa8c3603378b36abcb28abe9fcef4 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 20:49:14 +0200 Subject: [PATCH 063/128] Change internal type --- OTAnalytics/application/eventlist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index 44683f8d7..a01048584 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -53,7 +53,7 @@ class SceneEventListBuilder: """Create enter and leave scene events for track segments.""" def __init__(self) -> None: - self._events = EventDataset() + self._events: list[Event] = [] def add_enter_scene_events( self, segments: TrackSegmentDataset @@ -83,7 +83,7 @@ def build(self) -> EventDataset: Returns: EventDataset: complete event dataset """ - return EventDataset(self._events.get_events()) + return EventDataset(self._events) def _create_enter_scene_event(self, value: dict) -> None: event = self.__create_event( From 56d18b78e813d16e215a285c7cb9660ee3f69bef Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 20:55:41 +0200 Subject: [PATCH 064/128] Remove unused code --- OTAnalytics/domain/event.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/OTAnalytics/domain/event.py b/OTAnalytics/domain/event.py index b7fa831dc..82f41e7e9 100644 --- a/OTAnalytics/domain/event.py +++ b/OTAnalytics/domain/event.py @@ -657,14 +657,6 @@ def append(self, event: Event) -> None: """ self._events.append(event) - def get_events(self) -> list[Event]: - """Get the underlying list of events. - - Returns: - The list of events contained in this dataset. - """ - return self._events.copy() - def is_empty(self) -> bool: """Check if the dataset contains no events. From 8b8a4f3112c290aa6c213e6d45fe5489c3e6ed21 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 18 Sep 2025 21:05:57 +0200 Subject: [PATCH 065/128] Create abstract base for event data --- .../application/analysis/traffic_counting.py | 11 ++- OTAnalytics/application/eventlist.py | 3 +- .../use_cases/create_intersection_events.py | 17 +++-- OTAnalytics/domain/event.py | 67 +++++++++++++++++-- .../domain/track_dataset/track_dataset.py | 3 +- .../multiprocessing.py | 4 +- .../sequential.py | 4 +- .../analysis/test_traffic_counting.py | 8 ++- .../OTAnalytics/application/test_eventlist.py | 6 +- .../test_multiprocessing.py | 15 +++-- .../test_sequential.py | 4 +- 11 files changed, 108 insertions(+), 34 deletions(-) diff --git a/OTAnalytics/application/analysis/traffic_counting.py b/OTAnalytics/application/analysis/traffic_counting.py index ba94ea81e..ffb8ef0d2 100644 --- a/OTAnalytics/application/analysis/traffic_counting.py +++ b/OTAnalytics/application/analysis/traffic_counting.py @@ -14,7 +14,12 @@ from OTAnalytics.application.export_formats.export_mode import ExportMode from OTAnalytics.application.use_cases.create_events import CreateEvents from OTAnalytics.application.use_cases.section_repository import GetSectionsById -from OTAnalytics.domain.event import Event, EventDataset, EventRepository +from OTAnalytics.domain.event import ( + Event, + EventDataset, + EventRepository, + PythonEventDataset, +) from OTAnalytics.domain.flow import Flow, FlowRepository from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.types import EventType @@ -681,7 +686,7 @@ class FilterBySectionEnterEvent(RoadUserAssignerDecorator): """Decorator to filters events by event type section-enter.""" def assign(self, events: Iterable[Event], flows: list[Flow]) -> RoadUserAssignments: - section_enter_events = EventDataset( + section_enter_events = PythonEventDataset( [event for event in events if event.event_type == EventType.SECTION_ENTER] ) return super().assign(section_enter_events, flows) @@ -752,7 +757,7 @@ def __group_events_by_road_user( dict[tuple[RoadUserId, RoadUserType], EventDataset]: events grouped by user """ events_by_road_user: dict[tuple[RoadUserId, RoadUserType], EventDataset] = ( - defaultdict(EventDataset) + defaultdict(PythonEventDataset) ) sorted_events = sorted( events, key=lambda _event: _event.interpolated_occurrence diff --git a/OTAnalytics/application/eventlist.py b/OTAnalytics/application/eventlist.py index a01048584..5f3320c17 100644 --- a/OTAnalytics/application/eventlist.py +++ b/OTAnalytics/application/eventlist.py @@ -7,6 +7,7 @@ Event, EventDataset, ImproperFormattedFilename, + PythonEventDataset, ) from OTAnalytics.domain.geometry import ImageCoordinate, calculate_direction_vector from OTAnalytics.domain.track_dataset.track_dataset import ( @@ -83,7 +84,7 @@ def build(self) -> EventDataset: Returns: EventDataset: complete event dataset """ - return EventDataset(self._events) + return PythonEventDataset(self._events) def _create_enter_scene_event(self, value: dict) -> None: event = self.__create_event( diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index f098e2cd2..3bb14219c 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -4,7 +4,12 @@ RunIntersect, group_sections_by_offset, ) -from OTAnalytics.domain.event import EventBuilder, EventDataset, SectionEventBuilder +from OTAnalytics.domain.event import ( + EventBuilder, + EventDataset, + PythonEventDataset, + SectionEventBuilder, +) from OTAnalytics.domain.geometry import ( DirectionVector2D, RelativeOffsetCoordinate, @@ -42,7 +47,7 @@ def intersect( sections_grouped_by_offset = group_sections_by_offset( sections, EventType.SECTION_ENTER ) - event_dataset = EventDataset() + event_dataset = PythonEventDataset() for offset, section_group in sections_grouped_by_offset.items(): result = self.__do_intersect( track_dataset, section_group, offset, event_builder @@ -79,7 +84,7 @@ def intersect( sections_grouped_by_offset = group_sections_by_offset( sections, EventType.SECTION_ENTER ) - event_dataset = EventDataset() + event_dataset = PythonEventDataset() for offset, section_group in sections_grouped_by_offset.items(): result = self.__do_intersect( track_dataset, section_group, offset, event_builder @@ -172,7 +177,7 @@ def __do_intersect( events.append(event) section_currently_entered = entered - return EventDataset(events) + return PythonEventDataset(events) class RunCreateIntersectionEvents: @@ -191,7 +196,7 @@ def __init__( self._event_builder = event_builder def create(self) -> EventDataset: - event_dataset = EventDataset() + event_dataset = PythonEventDataset() line_sections, area_sections = separate_sections(self._sections) event_dataset.extend( self._intersect_line_section.intersect( @@ -232,7 +237,7 @@ def __call__(self, sections: Iterable[Section]) -> EventDataset: def _create_events(tracks: TrackDataset, sections: Iterable[Section]) -> EventDataset: - event_dataset = EventDataset() + event_dataset = PythonEventDataset() event_builder = SectionEventBuilder() create_intersection_events = RunCreateIntersectionEvents( diff --git a/OTAnalytics/domain/event.py b/OTAnalytics/domain/event.py index 82f41e7e9..e27ce44bb 100644 --- a/OTAnalytics/domain/event.py +++ b/OTAnalytics/domain/event.py @@ -596,7 +596,64 @@ def before_filter(date: datetime) -> Callable[[Event], bool]: return lambda actual: actual.occurrence <= date -class EventDataset: +class EventDataset(ABC): + """A collection of events.""" + + @abstractmethod + def __iter__(self) -> Iterator[Event]: + """Iterate over the events in the dataset.""" + raise NotImplementedError + + @abstractmethod + def __len__(self) -> int: + """Return the number of events in the dataset.""" + raise NotImplementedError + + @abstractmethod + def __add__(self, other: "EventDataset") -> "EventDataset": + """Add two EventDatasets together to create a new combined dataset. + + Args: + other: Another EventDataset to combine with this one. + + Returns: + A new EventDataset containing events from both datasets. + """ + raise NotImplementedError + + @abstractmethod + def __eq__(self, other: object) -> bool: + raise NotImplementedError + + @abstractmethod + def extend(self, other: "EventDataset") -> None: + """Extend this dataset with events from another dataset. + + Args: + other: Another EventDataset whose events will be added to this dataset. + """ + raise NotImplementedError + + @abstractmethod + def append(self, event: Event) -> None: + """Add a single event to the dataset. + + Args: + event: The event to add to the dataset. + """ + raise NotImplementedError + + @abstractmethod + def is_empty(self) -> bool: + """Check if the dataset contains no events. + + Returns: + True if the dataset is empty, False otherwise. + """ + raise NotImplementedError + + +class PythonEventDataset(EventDataset): """A dataset wrapper for a collection of events. This class wraps a list of events and provides methods for combining @@ -628,12 +685,12 @@ def __add__(self, other: "EventDataset") -> "EventDataset": Returns: A new EventDataset containing events from both datasets. """ - if not isinstance(other, EventDataset): + if not isinstance(other, PythonEventDataset): raise TypeError() - return EventDataset(self._events + other._events) + return PythonEventDataset(self._events + other._events) def __eq__(self, other: object) -> bool: - if not isinstance(other, EventDataset): + if not isinstance(other, PythonEventDataset): return False return self._events == other._events @@ -643,7 +700,7 @@ def extend(self, other: "EventDataset") -> None: Args: other: Another EventDataset whose events will be added to this dataset. """ - if isinstance(other, EventDataset): + if isinstance(other, PythonEventDataset): self._events.extend(other._events) else: # Support extending with a list of events for compatibility diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 3e2d6d812..ef1bcc884 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -7,6 +7,7 @@ Event, EventBuilder, EventDataset, + PythonEventDataset, SectionEventBuilder, ) from OTAnalytics.domain.geometry import ( @@ -581,7 +582,7 @@ def create_events( ) events.append(event_builder.create_event(detection)) - return EventDataset(events) + return PythonEventDataset(events) def _get_interpolated_occurrence( self, previous: datetime, current: datetime, relative_position: float diff --git a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py index 6515799e5..8306b9480 100644 --- a/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py +++ b/OTAnalytics/plugin_intersect_parallelization/multiprocessing.py @@ -3,7 +3,7 @@ from OTAnalytics.application.config import DEFAULT_NUM_PROCESSES from OTAnalytics.application.logger import logger -from OTAnalytics.domain.event import EventDataset +from OTAnalytics.domain.event import EventDataset, PythonEventDataset from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset @@ -48,7 +48,7 @@ def execute( def _combine_event_datasets( self, event_datasets: Iterable[EventDataset] ) -> EventDataset: - combined = EventDataset() + combined = PythonEventDataset() for dataset in event_datasets: combined.extend(dataset) return combined diff --git a/OTAnalytics/plugin_intersect_parallelization/sequential.py b/OTAnalytics/plugin_intersect_parallelization/sequential.py index 5afbbd809..b0b385c4f 100644 --- a/OTAnalytics/plugin_intersect_parallelization/sequential.py +++ b/OTAnalytics/plugin_intersect_parallelization/sequential.py @@ -1,6 +1,6 @@ from typing import Callable, Iterable, Sequence -from OTAnalytics.domain.event import EventDataset +from OTAnalytics.domain.event import EventDataset, PythonEventDataset from OTAnalytics.domain.intersect import IntersectParallelizationStrategy from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset @@ -18,7 +18,7 @@ def execute( intersect: Callable[[TrackDataset, Iterable[Section]], EventDataset], tasks: Sequence[tuple[TrackDataset, Iterable[Section]]], ) -> EventDataset: - event_dataset = EventDataset() + event_dataset = PythonEventDataset() for task in tasks: track_dataset, sections = task event_dataset.extend(intersect(track_dataset, sections)) diff --git a/tests/OTAnalytics/application/analysis/test_traffic_counting.py b/tests/OTAnalytics/application/analysis/test_traffic_counting.py index ec356ded2..088acc4ea 100644 --- a/tests/OTAnalytics/application/analysis/test_traffic_counting.py +++ b/tests/OTAnalytics/application/analysis/test_traffic_counting.py @@ -49,7 +49,7 @@ from OTAnalytics.application.export_formats.export_mode import OVERWRITE from OTAnalytics.application.use_cases.create_events import CreateEvents from OTAnalytics.application.use_cases.section_repository import GetSectionsById -from OTAnalytics.domain.event import Event, EventDataset, EventRepository +from OTAnalytics.domain.event import Event, EventRepository, PythonEventDataset from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate from OTAnalytics.domain.section import SectionId @@ -832,9 +832,11 @@ def test_assign_filters_by_section_enter_event(self) -> None: road_user_assigner = Mock(spec=RoadUserAssigner) decorator = FilterBySectionEnterEvent(road_user_assigner) - decorator.assign(EventDataset([enter_scene_event, section_enter_event]), [flow]) + decorator.assign( + PythonEventDataset([enter_scene_event, section_enter_event]), [flow] + ) road_user_assigner.assign.assert_called_once_with( - EventDataset([section_enter_event]), [flow] + PythonEventDataset([section_enter_event]), [flow] ) diff --git a/tests/OTAnalytics/application/test_eventlist.py b/tests/OTAnalytics/application/test_eventlist.py index 313ae9d92..26ad8de79 100644 --- a/tests/OTAnalytics/application/test_eventlist.py +++ b/tests/OTAnalytics/application/test_eventlist.py @@ -4,7 +4,7 @@ import pytest from OTAnalytics.application.eventlist import SceneActionDetector, SceneEventListBuilder -from OTAnalytics.domain.event import Event, EventDataset, EventType +from OTAnalytics.domain.event import Event, EventDataset, EventType, PythonEventDataset from OTAnalytics.domain.geometry import ( Coordinate, ImageCoordinate, @@ -179,7 +179,7 @@ def create_expected_events(track: Track) -> EventDataset: def create_expected_leave_scene_events(track: Track) -> EventDataset: occurrence = track.last_detection.occurrence event_coordinate = ImageCoordinate(track.last_detection.x, track.last_detection.y) - return EventDataset( + return PythonEventDataset( [ Event( road_user_id=track.id.id, @@ -208,7 +208,7 @@ def create_expected_enter_scene_events(track: Track) -> EventDataset: event_coordinate = ImageCoordinate(track.first_detection.x, track.first_detection.y) occurrence = track.first_detection.occurrence - return EventDataset( + return PythonEventDataset( [ Event( road_user_id=track.id.id, diff --git a/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py b/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py index ff41f13fe..3518744a5 100644 --- a/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py +++ b/tests/OTAnalytics/plugin_intersect_parallelization/test_multiprocessing.py @@ -3,7 +3,7 @@ import pytest -from OTAnalytics.domain.event import Event, EventDataset +from OTAnalytics.domain.event import Event, PythonEventDataset from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset from OTAnalytics.plugin_intersect_parallelization.multiprocessing import ( @@ -26,7 +26,7 @@ def test_execute(self, mock_pool_init: Mock) -> None: parallelizer = MultiprocessingIntersectParallelization() result = parallelizer.execute(cast(Callable, intersect), tasks) - assert result == EventDataset([event_1, event_2]) + assert result == PythonEventDataset([event_1, event_2]) mock_pool_instance.starmap.assert_called_once_with(intersect, tasks) @patch("OTAnalytics.plugin_intersect_parallelization.multiprocessing.Pool") @@ -37,7 +37,7 @@ def test_execute_sequentially(self, mock_pool_init: Mock) -> None: mock_pool_instance = mock_pool_init.return_value.__enter__.return_value intersect = Mock() - intersect.side_effect = [EventDataset([event_1, event_2])] + intersect.side_effect = [PythonEventDataset([event_1, event_2])] track_dataset: TrackDataset = Mock(spec=TrackDataset) section = Mock() task: tuple[TrackDataset, Iterable[Section]] = (track_dataset, [section]) @@ -45,7 +45,7 @@ def test_execute_sequentially(self, mock_pool_init: Mock) -> None: parallelizer = MultiprocessingIntersectParallelization(num_processes=1) result = parallelizer.execute(cast(Callable, intersect), [task]) - assert result == EventDataset([event_1, event_2]) + assert result == PythonEventDataset([event_1, event_2]) mock_pool_instance.starmap.assert_not_called() intersect.assert_called_once_with(track_dataset, [section]) @@ -53,10 +53,13 @@ def test_flatten_events(self) -> None: intersect = MultiprocessingIntersectParallelization() event_1 = Mock(spec=Event) event_2 = Mock(spec=Event) - events_to_flatten = [EventDataset([event_1]), EventDataset([event_2])] + events_to_flatten = [ + PythonEventDataset([event_1]), + PythonEventDataset([event_2]), + ] result = intersect._combine_event_datasets(events_to_flatten) - assert result == EventDataset([event_1, event_2]) + assert result == PythonEventDataset([event_1, event_2]) def test_set_num_processes(self) -> None: intersect = MultiprocessingIntersectParallelization(4) diff --git a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py index 61374d447..0194720be 100644 --- a/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py +++ b/tests/OTAnalytics/plugin_intersect_parallelization/test_sequential.py @@ -1,7 +1,7 @@ from typing import Callable, cast from unittest.mock import Mock, call -from OTAnalytics.domain.event import Event, EventDataset +from OTAnalytics.domain.event import Event, PythonEventDataset from OTAnalytics.domain.section import Section from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset from OTAnalytics.plugin_intersect_parallelization.sequential import SequentialIntersect @@ -25,7 +25,7 @@ def test_execute(self) -> None: sequential_intersect = SequentialIntersect() result = sequential_intersect.execute(cast(Callable, mock_intersect), tasks) - assert result == EventDataset([event_1, event_2]) + assert result == PythonEventDataset([event_1, event_2]) assert mock_intersect.call_args_list == [ call(first_track_dataset, sections), call(second_track_dataset, sections), From 66319c123ca5adb0307b889c9fd5cdf9bf3662cb Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 23 Sep 2025 09:57:31 +0200 Subject: [PATCH 066/128] Merge origin/main into feature branch customer-reqs/8596-speed-up-otanalytics-by-using-vectorized-operations Solved merge conflicts manually --- .pre-commit-config.yaml | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 35b9429ad..1312cbbb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,44 @@ repos: hooks: - id: mypy entry: mypy OTAnalytics tests --config-file=pyproject.toml - additional_dependencies: [] + additional_dependencies: + - av==13.0.0 + - black==24.8.0 + - customtkinter==5.2.2 + - fastapi==0.115.5 + - hatch-requirements-txt==0.4.1 + - ijson==3.3.0 + - interrogate==1.7.0 + - isort==5.13.2 + - matplotlib==3.10.0 + - more-itertools==10.5.0 + - mypy==1.16.1 + - nicegui==2.11.0 + - numpy==2.1.1 + - pandas-stubs==2.2.3.241126 + - pandas==2.2.3 + - polars==1.32.3 + - pre-commit==4.1.0 + - pyarrow==21.0.0 + - pytest-asyncio==0.24.0 + - pytest-benchmark==5.1.0 + - pytest-cov==6.0.0 + - pytest-lazy-fixtures==1.1.2 + - pytest==8.3.4 + - starlette==0.41.2 + - twine==6.1.0 + - types-PyYAML + - types-flake8 + - types-openpyxl + - types-pillow + - types-requests + - types-seaborn + - types-selenium + - types-shapely + - types-tqdm + - types-ujson + - uvicorn==0.32.1 + - yamllint==1.35.1 always_run: true pass_filenames: false - repo: https://github.com/adrienverge/yamllint.git From d990731ad45297eb760eebd42e1761eecc288954 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 23 Sep 2025 10:02:06 +0200 Subject: [PATCH 067/128] Fix failed test after merge --- tests/OTAnalytics/domain/track_dataset/test_track_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py b/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py index c19ae372d..395a7fdab 100644 --- a/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py +++ b/tests/OTAnalytics/domain/track_dataset/test_track_dataset.py @@ -53,8 +53,8 @@ def test_remove_by_original_ids( actual_dataset, removed_ids = target.remove_by_original_ids( PythonTrackIdSet([first_track_part_1.original_id]) ) - assert actual_dataset.track_ids == frozenset([uncut_track.id]) - assert removed_ids == frozenset( + assert actual_dataset.track_ids == PythonTrackIdSet([uncut_track.id]) + assert removed_ids == PythonTrackIdSet( [first_track_part_1.id, first_track_part_2.id] ) assert len(actual_dataset) == 1 From 0d238cea57fdf9f0bbd30c6c166f4c4551f22327 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 23 Sep 2025 10:03:10 +0200 Subject: [PATCH 068/128] Remove unused import --- OTAnalytics/application/use_cases/inside_cutting_section.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/OTAnalytics/application/use_cases/inside_cutting_section.py b/OTAnalytics/application/use_cases/inside_cutting_section.py index 48735f560..2421d8f83 100644 --- a/OTAnalytics/application/use_cases/inside_cutting_section.py +++ b/OTAnalytics/application/use_cases/inside_cutting_section.py @@ -1,10 +1,7 @@ -from typing import Iterable - from OTAnalytics.application.use_cases.section_repository import GetCuttingSections from OTAnalytics.application.use_cases.track_repository import GetAllTracks from OTAnalytics.domain.section import SectionListObserver, SectionRepositoryEvent -from OTAnalytics.domain.track import TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet, EmptyTrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackIdSet from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackListObserver, TrackRepositoryEvent from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet From d86b89f4814fd9c0c41e796f1416f3b1e681e98e Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 12:54:42 +0200 Subject: [PATCH 069/128] Move log output --- OTAnalytics/adapter_ui/dummy_viewmodel.py | 2 ++ OTAnalytics/plugin_ui/customtkinter_gui/frame_track_plotting.py | 2 -- .../nicegui_gui/pages/visualization_layers_form/layers_form.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/OTAnalytics/adapter_ui/dummy_viewmodel.py b/OTAnalytics/adapter_ui/dummy_viewmodel.py index 15b567273..aec99f105 100644 --- a/OTAnalytics/adapter_ui/dummy_viewmodel.py +++ b/OTAnalytics/adapter_ui/dummy_viewmodel.py @@ -1330,6 +1330,7 @@ def remove_flows(self) -> None: ) def create_events(self) -> None: + logger().info("Creating events") start_msg_popup = self._ui_factory.minimal_info_box( message="Create events...", initial_position=self.get_position(), @@ -1343,6 +1344,7 @@ def create_events(self) -> None: start_msg_popup.update_message(message="Creating events completed") sleep(1) start_msg_popup.close() + logger().info("Events created") def get_all_flow_ids(self) -> list[FlowId]: return [flow.id for flow in self.get_all_flows()] diff --git a/OTAnalytics/plugin_ui/customtkinter_gui/frame_track_plotting.py b/OTAnalytics/plugin_ui/customtkinter_gui/frame_track_plotting.py index 75f9def27..0481a25fa 100644 --- a/OTAnalytics/plugin_ui/customtkinter_gui/frame_track_plotting.py +++ b/OTAnalytics/plugin_ui/customtkinter_gui/frame_track_plotting.py @@ -7,7 +7,6 @@ AbstractFrameTrackPlotting, ) from OTAnalytics.adapter_ui.view_model import ViewModel -from OTAnalytics.application.logger import logger from OTAnalytics.application.plotting import Layer, LayerGroup from OTAnalytics.plugin_ui.customtkinter_gui.constants import PADX, PADY, STICKY from OTAnalytics.plugin_ui.customtkinter_gui.custom_containers import ( @@ -62,7 +61,6 @@ def _place_widgets(self) -> None: ) def _create_events(self) -> None: - logger().info("Creating events") self._viewmodel.create_events() def introduce_to_viewmodel(self) -> None: diff --git a/OTAnalytics/plugin_ui/nicegui_gui/pages/visualization_layers_form/layers_form.py b/OTAnalytics/plugin_ui/nicegui_gui/pages/visualization_layers_form/layers_form.py index 1b1a55dce..4b421a957 100644 --- a/OTAnalytics/plugin_ui/nicegui_gui/pages/visualization_layers_form/layers_form.py +++ b/OTAnalytics/plugin_ui/nicegui_gui/pages/visualization_layers_form/layers_form.py @@ -7,7 +7,6 @@ AbstractFrameTrackPlotting, ) from OTAnalytics.adapter_ui.view_model import ViewModel -from OTAnalytics.application.logger import logger from OTAnalytics.application.plotting import LayerGroup from OTAnalytics.application.resources.resource_manager import ( ResourceManager, @@ -54,7 +53,6 @@ def build(self) -> Self: return self def _create_events(self) -> None: - logger().info("Creating events") self._viewmodel.create_events() def update(self, name: str, start_date: Optional[datetime]) -> None: From f0a21a8e8ea42d5994c9d828c6f76480717e20d9 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 12:58:49 +0200 Subject: [PATCH 070/128] Extract IntersectionPointsDataset interface --- .../domain/track_dataset/track_dataset.py | 87 ++++++++++++++++++- .../test_create_intersection_events.py | 28 +++--- 2 files changed, 100 insertions(+), 15 deletions(-) diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index f90123bba..5dfc9c6c9 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -21,14 +21,22 @@ START_X: str = "start_x" START_Y: str = "start_y" +START_W: str = "start_w" +START_H: str = "start_h" START_OCCURRENCE: str = "start_occurrence" START_FRAME: str = "start_frame" START_VIDEO_NAME: str = "start_video_name" END_X: str = "end_x" END_Y: str = "end_y" +END_W: str = "end_w" +END_H: str = "end_h" END_OCCURRENCE: str = "end_occurrence" END_FRAME: str = "end_frame" END_VIDEO_NAME: str = "end_video_name" +CURRENT_X: str = "current_x" +CURRENT_Y: str = "current_y" +PREVIOUS_X: str = "previous_x" +PREVIOUS_Y: str = "previous_y" class TrackDoesNotExistError(Exception): @@ -238,7 +246,7 @@ def wrap_intersection_points( IntersectionPointsDataset: the intersection points dataset. """ intersection_data = self.intersection_points(sections, offset) - return IntersectionPointsDataset(intersection_data, self) + return PythonIntersectionPointsDataset(intersection_data, self) @abstractmethod def contained_by_sections( @@ -487,7 +495,82 @@ def __eq__(self, other: Any) -> bool: raise NotImplementedError -class IntersectionPointsDataset: +class IntersectionPointsDataset(ABC): + """Dataset containing intersection points between tracks and sections.""" + + @abstractmethod + def items( + self, + ) -> Iterator[tuple[TrackId, list[tuple[SectionId, IntersectionPoint]]]]: + """Iterate over track IDs and their intersection points. + + Returns: + Iterator yielding (track_id, intersection_points) tuples. + """ + raise NotImplementedError + + @abstractmethod + def keys(self) -> Iterator[TrackId]: + """Get all track IDs that have intersection points. + + Returns: + Iterator over track IDs. + """ + raise NotImplementedError + + @abstractmethod + def get(self, track_id: TrackId) -> list[tuple[SectionId, IntersectionPoint]]: + """Get intersection points for a specific track ID. + + Args: + track_id: The track ID to get intersection points for. + + Returns: + List of (section_id, intersection_point) tuples for the track. + """ + raise NotImplementedError + + @property + @abstractmethod + def empty(self) -> bool: + """Check if the dataset is empty. + + Returns: + True if no intersection points exist, False otherwise. + """ + raise NotImplementedError + + @abstractmethod + def __len__(self) -> int: + """Get the number of tracks with intersection points. + + Returns: + Number of tracks that have intersection points. + """ + raise NotImplementedError + + @abstractmethod + def __contains__(self, track_id: TrackId) -> bool: + """Check if a track ID has intersection points. + + Args: + track_id: The track ID to check. + + Returns: + True if the track has intersection points, False otherwise. + """ + raise NotImplementedError + + @abstractmethod + def create_events( + self, + offset: RelativeOffsetCoordinate, + event_builder: EventBuilder = SectionEventBuilder(), + ) -> EventDataset: + raise NotImplementedError + + +class PythonIntersectionPointsDataset(IntersectionPointsDataset): """Dataset containing intersection points between tracks and sections.""" def __init__( diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index 4f62bb5d0..affb01043 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -26,7 +26,7 @@ from OTAnalytics.domain.track import Detection, Track from OTAnalytics.domain.track_dataset.track_dataset import ( IntersectionPoint, - IntersectionPointsDataset, + PythonIntersectionPointsDataset, TrackDataset, ) from OTAnalytics.domain.types import EventType @@ -250,8 +250,8 @@ def test_case_track_line_section(track: Track) -> _TestCase: prev_detection = track.detections[ip.lower_index] detection = track.detections[ip.upper_index] track_dataset = Mock(spec=TrackDataset) - track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset( - {track.id: [(section.id, ip)]}, track_dataset + track_dataset.wrap_intersection_points.return_value = ( + PythonIntersectionPointsDataset({track.id: [(section.id, ip)]}, track_dataset) ) track_dataset.get_for.return_value = track expected_event_coord = _ExpectedEventCoord.from_detection( @@ -298,14 +298,16 @@ def test_case_closed_track_line_section( ) -> _TestCase: section = create_section([(0, 1.5), (3, 1.5)], SectionType.LINE) track_dataset = Mock(spec=TrackDataset) - track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset( - { - closed_track.id: [ - (section.id, IntersectionPoint(upper_index=2, relative_position=0)), - (section.id, IntersectionPoint(upper_index=4, relative_position=0)), - ] - }, - track_dataset, + track_dataset.wrap_intersection_points.return_value = ( + PythonIntersectionPointsDataset( + { + closed_track.id: [ + (section.id, IntersectionPoint(upper_index=2, relative_position=0)), + (section.id, IntersectionPoint(upper_index=4, relative_position=0)), + ] + }, + track_dataset, + ) ) track_dataset.get_for.return_value = closed_track expected_event_coords = [ @@ -326,8 +328,8 @@ def test_case_closed_track_line_section( def test_case_line_section_no_intersection(track: Track) -> _TestCase: section = create_section([(0, 0), (10, 0)], SectionType.LINE) track_dataset = Mock(spec=TrackDataset) - track_dataset.wrap_intersection_points.return_value = IntersectionPointsDataset( - {}, track_dataset + track_dataset.wrap_intersection_points.return_value = ( + PythonIntersectionPointsDataset({}, track_dataset) ) track_dataset.get_for.return_value = track From 3f81f3299fcfa4c0bac21460423a8ea22bf69259 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 13:00:51 +0200 Subject: [PATCH 071/128] Implement polars version of IntersectionPointsDataset --- .../plugin_datastore/polars_track_store.py | 15 +- .../polars_geometry_store.py | 395 +++++++++++++++++- 2 files changed, 392 insertions(+), 18 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 0e333f8cb..85d628817 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -39,6 +39,7 @@ START_Y, EmptyTrackIdSet, IntersectionPoint, + IntersectionPointsDataset, TrackDataset, TrackDoesNotExistError, TrackIdSet, @@ -216,7 +217,6 @@ def calculate(self, detections: pl.DataFrame) -> pl.DataFrame: track.H, track.FRAME, track.OCCURRENCE, - ottrk_dataformat.INPUT_FILE_PATH, track.INTERPOLATED_DETECTION, ottrk_dataformat.FIRST, ottrk_dataformat.FINISHED, @@ -382,7 +382,10 @@ def add_all(self, other: Iterable[Track]) -> "PolarsTrackDataset": # Get all tracks (existing + new) and assign classification combined_tracks = pl.concat( - [drop_row_id(self._dataset), drop_row_id(new_tracks_with_classification)] + [ + drop_row_id(self._dataset).select(COLUMNS), + drop_row_id(new_tracks_with_classification).select(COLUMNS), + ] ).sort(INDEX_NAMES) # Re-assign track classification to the combined dataset @@ -485,6 +488,12 @@ def intersection_points( geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.intersection_points(sections) + def wrap_intersection_points( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> IntersectionPointsDataset: + geometry_dataset = self._get_geometry_dataset_for(offset) + return geometry_dataset.wrap_intersection_points(sections) + def ids_inside(self, sections: list[Section]) -> TrackIdSet: result: TrackIdSet = PolarsTrackIdSet() for _section in sections: @@ -837,6 +846,8 @@ def _convert_tracks(tracks: Iterable[Track]) -> pl.DataFrame: for detection in current_track.detections: dto = detection.to_dict() dto[track.ORIGINAL_TRACK_ID] = current_track.original_id.id + dto[ottrk_dataformat.FIRST] = current_track.first_detection == detection + dto[ottrk_dataformat.FINISHED] = current_track.last_detection == detection prepared.append(dto) if not prepared: diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 05f1733dd..0d73cc8c4 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -1,16 +1,57 @@ import math from dataclasses import dataclass -from typing import Any, Iterable, Optional, Sequence +from typing import Any, Iterable, Iterator, Optional, Sequence import polars as pl -from OTAnalytics.domain import track +from OTAnalytics.domain import event, track from OTAnalytics.domain.common import DataclassValidation -from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.event import ( + SECTION_ID, + Event, + EventBuilder, + EventDataset, + SectionEventBuilder, +) +from OTAnalytics.domain.geometry import ( + Coordinate, + DirectionVector2D, + ImageCoordinate, + RelativeOffsetCoordinate, +) from OTAnalytics.domain.section import Section, SectionId, SectionType -from OTAnalytics.domain.track import OCCURRENCE, TRACK_ID, H, Track, TrackId, W, X, Y +from OTAnalytics.domain.track import ( + FRAME, + OCCURRENCE, + TRACK_CLASSIFICATION, + TRACK_ID, + VIDEO_NAME, + H, + Track, + TrackId, + W, + X, + Y, +) from OTAnalytics.domain.track_dataset.track_dataset import ( + CURRENT_X, + CURRENT_Y, + END_FRAME, + END_H, + END_OCCURRENCE, + END_VIDEO_NAME, + END_W, + END_X, + END_Y, + PREVIOUS_X, + PREVIOUS_Y, + START_H, + START_OCCURRENCE, + START_W, + START_X, + START_Y, IntersectionPoint, + IntersectionPointsDataset, TrackDataset, TrackGeometryDataset, TrackIdSet, @@ -18,18 +59,18 @@ from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSet +MAGNITUDE = "magnitude" + +SEGMENT_LENGTH_X = "segment_length_x" +SEGMENT_LENGTH_Y = "segment_length_y" +SEGMENT_LENGTH = "segment_length" +INTERSECTION_LENGTH_X = "intersection_length_x" +INTERSECTION_LENGTH_Y = "intersection_length_y" +INTERSECTION_LENGTH = "intersection_length" +RELATIVE_POSITION = "relative_position" + # Column names for track segments ROW_ID = "row_id" -START_OCCURRENCE = "start-occurrence" -END_OCCURRENCE = "end-occurrence" -START_X = "start-x" -START_Y = "start-y" -START_W = "start-w" -START_H = "start-h" -END_X = "end-x" -END_Y = "end-y" -END_W = "end-w" -END_H = "end-h" # Column names for intersections INTERSECTS = "intersects" @@ -44,6 +85,14 @@ UA = "ua" UB = "ub" +# Column names for events +DIRECTION_VECTOR_X = f"{event.DIRECTION_VECTOR}_X" +DIRECTION_VECTOR_Y = f"{event.DIRECTION_VECTOR}_Y" +EVENT_COORDINATE_X = f"{event.EVENT_COORDINATE}_X" +EVENT_COORDINATE_Y = f"{event.EVENT_COORDINATE}_Y" +INTERPOLATED_EVENT_COORDINATE_X = f"{event.INTERPOLATED_EVENT_COORDINATE}_X" +INTERPOLATED_EVENT_COORDINATE_Y = f"{event.INTERPOLATED_EVENT_COORDINATE}_Y" + @dataclass(frozen=True) class Polygon(DataclassValidation): @@ -100,6 +149,8 @@ def create_track_segments(df: pl.DataFrame) -> pl.DataFrame: pl.col(Y).alias(END_Y), pl.col(W).alias(END_W), pl.col(H).alias(END_H), + pl.col(FRAME).alias(END_FRAME), + pl.col(VIDEO_NAME).alias(END_VIDEO_NAME), # Start values (previous row within the same track) pl.col(OCCURRENCE).shift(1).over(TRACK_ID).alias(START_OCCURRENCE), pl.col(X).shift(1).over(TRACK_ID).alias(START_X), @@ -114,6 +165,9 @@ def create_track_segments(df: pl.DataFrame) -> pl.DataFrame: [ ROW_ID, TRACK_ID, + TRACK_CLASSIFICATION, + END_VIDEO_NAME, + END_FRAME, END_OCCURRENCE, END_X, END_Y, @@ -354,7 +408,10 @@ def find_line_intersections( ).select( [ ROW_ID, - track.TRACK_ID, + TRACK_ID, + TRACK_CLASSIFICATION, + END_VIDEO_NAME, + END_FRAME, START_X, START_Y, END_X, @@ -516,6 +573,172 @@ def get_section_offset(section: Section) -> RelativeOffsetCoordinate: raise ValueError(f"Section {section.id} has no offset") +class PolarsEventDataset(EventDataset): + + def __init__(self, events: Optional[pl.DataFrame] = None) -> None: + if events is None: + self._events = pl.DataFrame() + else: + self._events = events + + def __iter__(self) -> Iterator[Event]: + for row in self._events.iter_rows(named=True): + yield Event( + road_user_id=row[event.ROAD_USER_ID], + road_user_type=row[event.ROAD_USER_TYPE], + hostname=row[event.HOSTNAME], + occurrence=row[event.OCCURRENCE], + frame_number=row[event.FRAME_NUMBER], + section_id=( + SectionId(row[event.SECTION_ID]) if row[event.SECTION_ID] else None + ), + event_coordinate=ImageCoordinate( + row[EVENT_COORDINATE_X], row[EVENT_COORDINATE_Y] + ), + event_type=EventType.parse(row[event.EVENT_TYPE]), + direction_vector=DirectionVector2D( + row[DIRECTION_VECTOR_X], row[DIRECTION_VECTOR_Y] + ), + video_name=row[event.VIDEO_NAME], + interpolated_occurrence=row[event.INTERPOLATED_OCCURRENCE], + interpolated_event_coordinate=ImageCoordinate( + row[INTERPOLATED_EVENT_COORDINATE_X], + row[INTERPOLATED_EVENT_COORDINATE_Y], + ), + ) + + def __len__(self) -> int: + return len(self._events) + + def __add__(self, other: "EventDataset") -> "EventDataset": + raise NotImplementedError + + def __eq__(self, other: object) -> bool: + raise NotImplementedError + + def extend(self, other: "EventDataset") -> None: + raise NotImplementedError + + def append(self, event: Event) -> None: + raise NotImplementedError + + def is_empty(self) -> bool: + return self._events.is_empty() + + +class PolarsIntersectionPointsDataset(IntersectionPointsDataset): + + def __init__(self, points: Optional[pl.DataFrame] = None) -> None: + if points is None: + self._points = pl.DataFrame() + else: + self._points = points + + def items( + self, + ) -> Iterator[tuple[TrackId, list[tuple[SectionId, IntersectionPoint]]]]: + raise NotImplementedError + + def keys(self) -> Iterator[TrackId]: + raise NotImplementedError + + def get(self, track_id: TrackId) -> list[tuple[SectionId, IntersectionPoint]]: + raise NotImplementedError + + @property + def empty(self) -> bool: + return self._points.is_empty() + + def __len__(self) -> int: + return len(self._points) + + def __contains__(self, track_id: TrackId) -> bool: + raise NotImplementedError + + def create_events( + self, + offset: RelativeOffsetCoordinate, + event_builder: EventBuilder = SectionEventBuilder(), + ) -> EventDataset: + if self.empty: + return PolarsEventDataset() + events = ( + self._points.with_columns( + [ + pl.col(TRACK_ID).alias(event.ROAD_USER_ID), + pl.col(TRACK_CLASSIFICATION).alias(event.ROAD_USER_TYPE), + pl.col(END_VIDEO_NAME).alias( + event.HOSTNAME + ), # TODO extract hostname + pl.col(END_OCCURRENCE).alias(event.OCCURRENCE), + pl.col(END_FRAME).alias(event.FRAME_NUMBER), + pl.col(SECTION_ID).alias(event.SECTION_ID), + pl.col(END_X).alias(EVENT_COORDINATE_X), + pl.col(END_Y).alias(EVENT_COORDINATE_Y), + pl.lit(EventType.SECTION_ENTER.value).alias(event.EVENT_TYPE), + (pl.col(PREVIOUS_Y) - pl.col(PREVIOUS_X)).alias(DIRECTION_VECTOR_X), + (pl.col(CURRENT_Y) - pl.col(CURRENT_X)).alias(DIRECTION_VECTOR_Y), + pl.col(END_VIDEO_NAME).alias(event.VIDEO_NAME), + ( + pl.col(START_OCCURRENCE) + + pl.col(RELATIVE_POSITION) + * (pl.col(END_OCCURRENCE) - pl.col(START_OCCURRENCE)) + ).alias(event.INTERPOLATED_OCCURRENCE), + ( + pl.col(PREVIOUS_X) + + pl.col(RELATIVE_POSITION) + * (pl.col(CURRENT_X) - pl.col(PREVIOUS_X)) + ).alias(INTERPOLATED_EVENT_COORDINATE_X), + ( + pl.col(PREVIOUS_Y) + + pl.col(RELATIVE_POSITION) + * (pl.col(CURRENT_Y) - pl.col(PREVIOUS_Y)) + ).alias(INTERPOLATED_EVENT_COORDINATE_Y), + ] + ) + .with_columns( + [ + ( + pl.col(DIRECTION_VECTOR_X) ** 2 + + pl.col(DIRECTION_VECTOR_Y) ** 2 + ).alias(MAGNITUDE), + ] + ) + .with_columns( + [ + pl.when(pl.col(MAGNITUDE) == 0) + .then(pl.lit(0)) + .otherwise(pl.col(DIRECTION_VECTOR_X) / pl.col(MAGNITUDE)) + .alias(DIRECTION_VECTOR_X), + pl.when(pl.col(MAGNITUDE) == 0) + .then(pl.lit(0)) + .otherwise(pl.col(DIRECTION_VECTOR_Y) / pl.col(MAGNITUDE)) + .alias(DIRECTION_VECTOR_Y), + ] + ) + .select( + [ + event.ROAD_USER_ID, + event.ROAD_USER_TYPE, + event.HOSTNAME, + event.OCCURRENCE, + event.FRAME_NUMBER, + event.SECTION_ID, + EVENT_COORDINATE_X, + EVENT_COORDINATE_Y, + event.EVENT_TYPE, + DIRECTION_VECTOR_X, + DIRECTION_VECTOR_Y, + event.VIDEO_NAME, + event.INTERPOLATED_OCCURRENCE, + INTERPOLATED_EVENT_COORDINATE_X, + INTERPOLATED_EVENT_COORDINATE_Y, + ] + ) + ) + return PolarsEventDataset(events) + + class PolarsTrackGeometryDataset(TrackGeometryDataset): _segments_df: pl.DataFrame @@ -907,9 +1130,149 @@ def intersection_points( if track_id not in result: result[track_id] = [] result[track_id].append((section.id, intersection_point)) - return result + def wrap_intersection_points( + self, sections: list[Section] + ) -> IntersectionPointsDataset: + """Return the intersection points from tracks and the given sections. + + Args: + sections (list[Section]): the sections to intersect with. + + Returns: + IntersectionPointsDataset: the intersection points. + """ + if self.empty or not sections: + return PolarsIntersectionPointsDataset() + + # Process only line sections + # (area sections don't have specific intersection points) + line_sections = [ + section + for section in sections + if section.get_type() == SectionType.LINE + or section.get_type() == SectionType.CUTTING + ] + + if not line_sections: + return PolarsIntersectionPointsDataset() + + result_df: list[pl.DataFrame] = [] + # For each line section, find intersections with track segments + for section in line_sections: + # Get the coordinates of the section + coordinates = section.get_coordinates() + offset = get_section_offset(section) + + # Process each leg of the section (consecutive pair of coordinates) + for i in range(len(coordinates) - 1): + start_x, start_y = coordinates[i].x, coordinates[i].y + end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y + + # Find intersections with this leg of the line + intersections = find_line_intersections( + self._segments_df, + section.id.serialize(), + start_x, + start_y, + end_x, + end_y, + offset, + ) + + # Filter to only include segments that intersect with the line + intersecting_segments = intersections.filter(pl.col(INTERSECTS)) + + intersection_points = ( + intersecting_segments.with_columns( + [ + (pl.col(END_X) + pl.col(END_W) * offset.x).alias(CURRENT_X), + (pl.col(END_Y) + pl.col(END_H) * offset.y).alias(CURRENT_Y), + (pl.col(START_X) + pl.col(START_W) * offset.x).alias( + PREVIOUS_X + ), + (pl.col(START_Y) + pl.col(START_H) * offset.y).alias( + PREVIOUS_Y + ), + ] + ) + .with_columns( + [ + (pl.col(CURRENT_X) - pl.col(PREVIOUS_X)).alias( + SEGMENT_LENGTH_X + ), + (pl.col(CURRENT_Y) - pl.col(PREVIOUS_Y)).alias( + SEGMENT_LENGTH_Y + ), + ] + ) + .with_columns( + [ + ( + pl.col(SEGMENT_LENGTH_X) ** 2 + + pl.col(SEGMENT_LENGTH_Y) ** 2 + ) + .sqrt() + .alias(SEGMENT_LENGTH) + ] + ) + .with_columns( + [ + ( + pl.col(INTERSECTION_X) + - (pl.col(START_X) + pl.col(START_W) * offset.x) + ).alias(INTERSECTION_LENGTH_X), + ( + pl.col(INTERSECTION_Y) + - (pl.col(START_Y) + pl.col(START_H) * offset.y) + ).alias(INTERSECTION_LENGTH_Y), + ] + ) + .with_columns( + [ + ( + pl.col(INTERSECTION_LENGTH_X) ** 2 + + pl.col(INTERSECTION_LENGTH_Y) ** 2 + ) + .sqrt() + .alias(INTERSECTION_LENGTH) + ] + ) + .with_columns( + [ + pl.when( + (pl.col(SEGMENT_LENGTH_X) == 0) + & (pl.col(SEGMENT_LENGTH_Y) == 0) + ) + .then(None) + .otherwise( + pl.col(INTERSECTION_LENGTH) / pl.col(SEGMENT_LENGTH) + ) + .alias(RELATIVE_POSITION) + ] + ) + .filter(pl.col(RELATIVE_POSITION).is_not_null()) + .drop( + [ + SEGMENT_LENGTH_X, + SEGMENT_LENGTH_Y, + SEGMENT_LENGTH, + INTERSECTION_LENGTH_X, + INTERSECTION_LENGTH_Y, + INTERSECTION_LENGTH, + ] + ) + .with_columns(pl.lit(section.id.id).alias(SECTION_ID)) + ) + + if len(intersecting_segments) > 0: + result_df.append(intersection_points) + + if result_df: + return PolarsIntersectionPointsDataset(pl.concat(result_df)) + return PolarsIntersectionPointsDataset() + def contained_by_sections( self, sections: list[Section] ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: From 0a9424058e3470ec2a2e8a6964dcd255b5cab74f Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 13:01:14 +0200 Subject: [PATCH 072/128] Use polars geometry set --- OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py index c956f71ff..33c99207f 100644 --- a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py +++ b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py @@ -4,8 +4,8 @@ from OTAnalytics.application.datastore import TrackParseResult from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - PandasTrackGeometryDataset, +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + PolarsTrackGeometryDataset, ) from OTAnalytics.plugin_datastore.track_store import ( PandasByMaxConfidence, @@ -21,7 +21,7 @@ def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: """Create a track geometry factory for PandasTrackDataset.""" - return PandasTrackGeometryDataset.from_track_dataset + return PolarsTrackGeometryDataset.from_track_dataset def create_ottrk_parser() -> OttrkParser: From 5ead3508d4922a5c564d92732482231a2cdc4168 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 13:01:38 +0200 Subject: [PATCH 073/128] Convert pandas test to polars test --- ...store.py => test_polars_geometry_store.py} | 136 +++++++++++------- 1 file changed, 81 insertions(+), 55 deletions(-) rename tests/OTAnalytics/plugin_datastore/track_geometry_store/{test_pandas_geometry_store.py => test_polars_geometry_store.py} (71%) diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_polars_geometry_store.py similarity index 71% rename from tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py rename to tests/OTAnalytics/plugin_datastore/track_geometry_store/test_polars_geometry_store.py index 15fb7132e..cf5295f1f 100644 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_pandas_geometry_store.py +++ b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_polars_geometry_store.py @@ -1,13 +1,14 @@ from datetime import datetime -import pandas as pd +import polars import pytest -from pandas import DataFrame +from polars import DataFrame from pytest import approx from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.track import H, W -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( +from OTAnalytics.domain.track import FRAME, TRACK_CLASSIFICATION, VIDEO_NAME, H, W +from OTAnalytics.domain.track_dataset.track_dataset import END_FRAME, END_VIDEO_NAME +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( END_H, END_OCCURRENCE, END_W, @@ -19,6 +20,7 @@ INTERSECTS, INTERSECTS_POLYGON, OCCURRENCE, + ROW_ID, START_H, START_OCCURRENCE, START_W, @@ -40,14 +42,18 @@ def test_find_line_intersections_empty_df() -> None: result = find_line_intersections( df, "line1", 0.0, 0.0, 10.0, 10.0, RelativeOffsetCoordinate(0.0, 0.0) ) - assert result.empty + assert result.is_empty() def test_find_line_intersections_no_intersections() -> None: """Test with segments that don't intersect with the line.""" # Create test data segments_data = { + ROW_ID: [1, 2], TRACK_ID: ["track1", "track2"], + TRACK_CLASSIFICATION: ["car", "car"], + END_VIDEO_NAME: ["video1", "video1"], + END_FRAME: [2, 3], START_X: [10.0, 100.0], START_Y: [15.0, 150.0], END_X: [20.0, 110.0], @@ -86,10 +92,10 @@ def test_find_line_intersections_no_intersections() -> None: ) # Check that no segments intersect - assert not result[INTERSECTS].any() - assert (result[INTERSECTION_X].isna()).all() - assert (result[INTERSECTION_Y].isna()).all() - assert (result[INTERSECTION_LINE_ID].isna()).all() + assert not result.get_column(INTERSECTS).any() + assert (result.get_column(INTERSECTION_X).is_nan()).all() + assert (result.get_column(INTERSECTION_Y).is_nan()).all() + assert (result.get_column(INTERSECTION_LINE_ID).is_null()).all() @pytest.mark.parametrize( @@ -112,7 +118,11 @@ def test_find_line_intersections_with_intersections( """Test with segments that intersect with the line.""" # Create test data segments_data = { + ROW_ID: [1, 2], TRACK_ID: ["track1", "track2"], + TRACK_CLASSIFICATION: ["car", "car"], + END_VIDEO_NAME: ["video1", "video1"], + END_FRAME: [2, 3], START_X: [60.0, 100.0], START_Y: [0.0, 150.0], END_X: [0.0, 110.0], @@ -145,23 +155,27 @@ def test_find_line_intersections_with_intersections( ) # Check that the first segment intersects - assert result.iloc[0][INTERSECTS] - assert result.iloc[0][INTERSECTION_X] == approx(expected_intersection) - assert result.iloc[0][INTERSECTION_Y] == approx(expected_intersection) - assert result.iloc[0][INTERSECTION_LINE_ID] == line_id + assert result.row(0, named=True)[INTERSECTS] + assert result.row(0, named=True)[INTERSECTION_X] == approx(expected_intersection) + assert result.row(0, named=True)[INTERSECTION_Y] == approx(expected_intersection) + assert result.row(0, named=True)[INTERSECTION_LINE_ID] == line_id # Check that the second segment doesn't intersect - assert not result.iloc[1][INTERSECTS] - assert pd.isna(result.iloc[1][INTERSECTION_X]) - assert pd.isna(result.iloc[1][INTERSECTION_Y]) - assert pd.isna(result.iloc[1][INTERSECTION_LINE_ID]) + assert not result.row(1, named=True)[INTERSECTS] + assert (result.slice(1, 1).get_column(INTERSECTION_X).is_nan()).all() + assert (result.slice(1, 1).get_column(INTERSECTION_Y).is_nan()).all() + assert (result.slice(1, 1).get_column(INTERSECTION_LINE_ID).is_null()).all() def test_find_line_intersections_multiple_intersections() -> None: """Test with multiple segments intersecting with the line.""" # Create test data segments_data = { + ROW_ID: [1, 2, 3], TRACK_ID: ["track1", "track2", "track3"], + TRACK_CLASSIFICATION: ["car", "car", "car"], + END_VIDEO_NAME: ["video1", "video1", "video1"], + END_FRAME: [2, 3, 4], START_X: [20.0, 100.0, 30.0], START_Y: [10.0, 10.0, 40.0], END_X: [10.0, 110.0, 40.0], @@ -202,17 +216,17 @@ def test_find_line_intersections_multiple_intersections() -> None: ) # Check that the first and third segments intersect - assert result.iloc[0][INTERSECTS] - assert result.iloc[0][INTERSECTION_X] == approx(15.0) - assert result.iloc[0][INTERSECTION_Y] == approx(15.0) - assert result.iloc[0][INTERSECTION_LINE_ID] == line_id + assert result.row(0, named=True)[INTERSECTS] + assert result.row(0, named=True)[INTERSECTION_X] == approx(15.0) + assert result.row(0, named=True)[INTERSECTION_Y] == approx(15.0) + assert result.row(0, named=True)[INTERSECTION_LINE_ID] == line_id - assert not result.iloc[1][INTERSECTS] + assert not result.row(1, named=True)[INTERSECTS] - assert result.iloc[2][INTERSECTS] - assert result.iloc[2][INTERSECTION_X] == approx(35.0) - assert result.iloc[2][INTERSECTION_Y] == approx(35.0) - assert result.iloc[2][INTERSECTION_LINE_ID] == line_id + assert result.row(2, named=True)[INTERSECTS] + assert result.row(2, named=True)[INTERSECTION_X] == approx(35.0) + assert result.row(2, named=True)[INTERSECTION_Y] == approx(35.0) + assert result.row(2, named=True)[INTERSECTION_LINE_ID] == line_id # Check the count of intersecting segments assert result[INTERSECTS].sum() == 2 @@ -222,19 +236,23 @@ def test_create_track_segments_empty_df() -> None: """Test that an empty DataFrame returns an empty DataFrame.""" df = DataFrame() result = create_track_segments(df) - assert result.empty + assert result.is_empty() def test_create_track_segments_single_track() -> None: """Test creating segments from a single track with multiple points.""" # Create test data data = { + ROW_ID: [1, 2, 3], TRACK_ID: ["track1", "track1", "track1"], + TRACK_CLASSIFICATION: ["car", "car", "car"], OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 2), ], + FRAME: [1, 2, 3], + VIDEO_NAME: ["video1", "video1", "video1"], X: [10.0, 20.0, 30.0], Y: [15.0, 25.0, 35.0], W: [10.0, 20.0, 30.0], @@ -249,47 +267,51 @@ def test_create_track_segments_single_track() -> None: assert len(result) == 2 # Should have 2 segments for 3 points # Check first segment - assert result.iloc[0][TRACK_ID] == "track1" - assert result.iloc[0][START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 0) - assert result.iloc[0][END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) - assert result.iloc[0][START_X] == 10.0 - assert result.iloc[0][START_Y] == 15.0 - assert result.iloc[0][START_W] == 10.0 - assert result.iloc[0][START_H] == 8.0 - assert result.iloc[0][END_X] == 20.0 - assert result.iloc[0][END_Y] == 25.0 - assert result.iloc[0][END_W] == 20.0 - assert result.iloc[0][END_H] == 16.0 + assert result.row(0, named=True)[TRACK_ID] == "track1" + assert result.row(0, named=True)[START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 0) + assert result.row(0, named=True)[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) + assert result.row(0, named=True)[START_X] == 10.0 + assert result.row(0, named=True)[START_Y] == 15.0 + assert result.row(0, named=True)[START_W] == 10.0 + assert result.row(0, named=True)[START_H] == 8.0 + assert result.row(0, named=True)[END_X] == 20.0 + assert result.row(0, named=True)[END_Y] == 25.0 + assert result.row(0, named=True)[END_W] == 20.0 + assert result.row(0, named=True)[END_H] == 16.0 # Check second segment - assert result.iloc[1][TRACK_ID] == "track1" - assert result.iloc[1][START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) - assert result.iloc[1][END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 2) - assert result.iloc[1][START_X] == 20.0 - assert result.iloc[1][START_Y] == 25.0 - assert result.iloc[1][START_W] == 20.0 - assert result.iloc[1][START_H] == 16.0 - assert result.iloc[1][END_X] == 30.0 - assert result.iloc[1][END_Y] == 35.0 - assert result.iloc[1][END_W] == 30.0 - assert result.iloc[1][END_H] == 24.0 + assert result.row(1, named=True)[TRACK_ID] == "track1" + assert result.row(1, named=True)[START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) + assert result.row(1, named=True)[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 2) + assert result.row(1, named=True)[START_X] == 20.0 + assert result.row(1, named=True)[START_Y] == 25.0 + assert result.row(1, named=True)[START_W] == 20.0 + assert result.row(1, named=True)[START_H] == 16.0 + assert result.row(1, named=True)[END_X] == 30.0 + assert result.row(1, named=True)[END_Y] == 35.0 + assert result.row(1, named=True)[END_W] == 30.0 + assert result.row(1, named=True)[END_H] == 24.0 def test_create_track_segments_multiple_tracks() -> None: """Test creating segments from multiple tracks.""" # Create test data data = { + ROW_ID: [1, 2, 3, 4], TRACK_ID: ["track1", "track1", "track2", "track2"], + TRACK_CLASSIFICATION: ["car", "car", "car", "car"], OCCURRENCE: [ datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 1), datetime(2023, 1, 1, 10, 0, 0), datetime(2023, 1, 1, 10, 0, 1), ], + FRAME: [1, 2, 1, 2], X: [10.0, 20.0, 100.0, 110.0], Y: [15.0, 25.0, 150.0, 160.0], W: [5.0, 10.0, 20.0, 40.0], H: [8.0, 16.0, 30.0, 60.0], + VIDEO_NAME: ["video1", "video1", "video1", "video1"], } df = DataFrame(data) @@ -300,7 +322,7 @@ def test_create_track_segments_multiple_tracks() -> None: assert len(result) == 2 # Should have 2 segments (1 for each track) # Check track1 segment - track1_segment = result[result[TRACK_ID] == "track1"].iloc[0] + track1_segment = result.filter(polars.col(TRACK_ID) == "track1").row(0, named=True) assert track1_segment[START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 0) assert track1_segment[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) assert track1_segment[START_X] == 10.0 @@ -313,7 +335,7 @@ def test_create_track_segments_multiple_tracks() -> None: assert track1_segment[END_H] == 16.0 # Check track2 segment - track2_segment = result[result[TRACK_ID] == "track2"].iloc[0] + track2_segment = result.filter(polars.col(TRACK_ID) == "track2").row(0, named=True) assert track2_segment[START_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 0) assert track2_segment[END_OCCURRENCE] == datetime(2023, 1, 1, 10, 0, 1) assert track2_segment[START_X] == 100.0 @@ -341,7 +363,7 @@ def test_check_polygon_intersections_empty_df() -> None: result = check_polygon_intersections( df, polygon, RelativeOffsetCoordinate(0.0, 0.0) ) - assert result.empty + assert result.is_empty() def test_check_polygon_intersections_no_intersections() -> None: @@ -392,7 +414,11 @@ def test_check_polygon_intersections_with_intersections() -> None: """Test with segments that intersect with the polygon.""" # Create test data segments_data = { + ROW_ID: [1, 2, 3], TRACK_ID: ["track1", "track2", "track3"], + TRACK_CLASSIFICATION: ["car", "car", "car"], + FRAME: [1, 2, 3], + VIDEO_NAME: ["video1", "video1", "video1"], START_X: [5.0, 100.0, 0.0], START_Y: [5.0, 150.0, 5.0], END_X: [15.0, 110.0, 25.0], @@ -431,9 +457,9 @@ def test_check_polygon_intersections_with_intersections() -> None: ) # Check that the first and third segments intersect with the polygon - assert result.iloc[0][INTERSECTS_POLYGON] - assert not result.iloc[1][INTERSECTS_POLYGON] - assert result.iloc[2][INTERSECTS_POLYGON] + assert result.row(0, named=True)[INTERSECTS_POLYGON] + assert not result.row(1, named=True)[INTERSECTS_POLYGON] + assert result.row(2, named=True)[INTERSECTS_POLYGON] # Check the count of intersecting segments assert result[INTERSECTS_POLYGON].sum() == 2 From b5c17800087f14c18557521d9ff40f12ecdc0cd0 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 13:01:50 +0200 Subject: [PATCH 074/128] Remove unused pandas implementation --- .../pandas_geometry_store.py | 996 ------------------ .../test_geometry_store_comparison.py | 495 --------- 2 files changed, 1491 deletions(-) delete mode 100644 OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py delete mode 100644 tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py deleted file mode 100644 index b2361d944..000000000 --- a/OTAnalytics/plugin_datastore/track_geometry_store/pandas_geometry_store.py +++ /dev/null @@ -1,996 +0,0 @@ -import math -from collections import defaultdict -from dataclasses import dataclass -from typing import Any, Iterable, Optional, Sequence - -import shapely -from pandas import DataFrame, concat -from shapely.creation import prepare -from shapely.geometry.base import BaseGeometry -from shapely.vectorized import contains - -from OTAnalytics.domain import track -from OTAnalytics.domain.common import DataclassValidation -from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.section import Section, SectionId, SectionType -from OTAnalytics.domain.track import H, Track, TrackId, W -from OTAnalytics.domain.track_dataset.track_dataset import ( - IntersectionPoint, - TrackDataset, - TrackGeometryDataset, - TrackIdSet, -) -from OTAnalytics.domain.types import EventType -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet -from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset - -# Column names for track points -TRACK_ID = track.TRACK_ID -OCCURRENCE = track.OCCURRENCE -X = "x" -Y = "y" - -# Column names for track segments -START_OCCURRENCE = "start-occurrence" -END_OCCURRENCE = "end-occurrence" -START_X = "start-x" -START_Y = "start-y" -START_W = "start-w" -START_H = "start-h" -END_X = "end-x" -END_Y = "end-y" -END_W = "end-w" -END_H = "end-h" - -# Column names for intersections -INTERSECTS = "intersects" -INTERSECTION_X = "intersection_x" -INTERSECTION_Y = "intersection_y" -INTERSECTION_LINE_ID = "intersection_line_id" -INTERSECTS_POLYGON = "intersects-polygon" - -# Column names for intersection parameters -DENOMINATOR = "denominator" -NON_PARALLEL = "non_parallel" -UA = "ua" -UB = "ub" - - -@dataclass(frozen=True) -class Polygon(DataclassValidation): - """A polygon consists of an ordered sequence of coordinates. - - Note: - The coordinates are not required to form a closed chain in this representation. - - Args: - coordinates (list[Coordinate]): The coordinates defining the polygon. Must - contain at least two coordinates. - - Raises: - ValueError: If fewer than two coordinates are provided. - """ - - coordinates: list[Coordinate] - - def _validate(self) -> None: - if len(self.coordinates) < 2: - raise ValueError( - ( - "Number of coordinates to define a valid polygon must be " - f"greater equal two, but is {len(self.coordinates)}" - ) - ) - - -def create_track_segments(df: DataFrame) -> DataFrame: - """ - Create track segments from a DataFrame of track points. - - A track segment connects two consecutive points of a track. - - Args: - df (DataFrame): DataFrame with columns TRACK_ID, OCCURRENCE, X, Y, W, H. - - Returns: - DataFrame: DataFrame with columns TRACK_ID, START_OCCURRENCE, END_OCCURRENCE, - START_X, START_Y, START_W, START_H, END_X, END_Y, END_W, END_H. - """ - if df.empty: - return DataFrame() - - # Create MultiIndex and sort once - if TRACK_ID in df.columns or OCCURRENCE in df.columns: - df_indexed = df.reset_index().set_index([TRACK_ID, OCCURRENCE]).sort_index() - else: - df_indexed = df.sort_index() - df_indexed[END_OCCURRENCE] = df_indexed.index.get_level_values(1) - - # Group by level 0 (TRACK_ID) - this is very fast on sorted MultiIndex - grouped = df_indexed.groupby(level=0) - - # Create segments DataFrame - segments = DataFrame(index=df_indexed.index) - segments[TRACK_ID] = df_indexed.index.get_level_values(0) - segments[END_OCCURRENCE] = df_indexed[END_OCCURRENCE] - segments[END_X] = df_indexed[X] - segments[END_Y] = df_indexed[Y] - segments[END_W] = df_indexed[W] - segments[END_H] = df_indexed[H] - - # Shift operations are faster on sorted groups - segments[START_OCCURRENCE] = grouped[END_OCCURRENCE].shift(1) - segments[START_X] = grouped[X].shift(1) - segments[START_Y] = grouped[Y].shift(1) - segments[START_W] = grouped[W].shift(1) - segments[START_H] = grouped[H].shift(1) - - # Remove NaN rows and reset index - segments = segments.dropna().reset_index(drop=True) - - return segments - - -def calculate_intersection_parameters( - segments_df: DataFrame, - line_x1: float, - line_y1: float, - line_x2: float, - line_y2: float, - offset: RelativeOffsetCoordinate, -) -> DataFrame: - """ - Calculate parameters needed for line intersection calculations. - - Args: - segments_df (DataFrame): DataFrame with track segments - line_x1 (float): X-coordinate of the line's start point - line_y1 (float): Y-coordinate of the line's start point - line_x2 (float): X-coordinate of the line's end point - line_y2 (float): Y-coordinate of the line's end point - - Returns: - DataFrame: A DataFrame containing the following columns: - - DENOMINATOR: Denominator values for intersection calculations - - NON_PARALLEL: Boolean mask for non-parallel segments - - UA: Parameter ua for intersection point calculation - - UB: Parameter ub for intersection point calculation - """ - # Initialize result DataFrame - result_df = DataFrame(index=segments_df.index) - - # Apply offset on-the-fly (no mutation/copy) - sxa = segments_df[START_X] + segments_df[START_W] * offset.x - sya = segments_df[START_Y] + segments_df[START_H] * offset.y - exa = segments_df[END_X] + segments_df[END_W] * offset.x - eya = segments_df[END_Y] + segments_df[END_H] * offset.y - - # Calculate denominator for all segments at once - result_df[DENOMINATOR] = (line_y2 - line_y1) * (exa - sxa) - (line_x2 - line_x1) * ( - eya - sya - ) - - # Filter out segments where lines are parallel (denominator == 0) - result_df[NON_PARALLEL] = result_df[DENOMINATOR] != 0 - - # Initialize ua and ub columns - result_df[UA] = None - result_df[UB] = None - - if result_df[NON_PARALLEL].any(): - # Calculate the parameters for the intersection points - ua_values = ( - (line_x2 - line_x1) * (sya - line_y1) - - (line_y2 - line_y1) * (sxa - line_x1) - ) / result_df[DENOMINATOR] - - ub_values = ( - (exa - sxa) * (sya - line_y1) - (eya - sya) * (sxa - line_x1) - ) / result_df[DENOMINATOR] - - # Update ua and ub columns for non-parallel segments - result_df.loc[result_df[NON_PARALLEL], UA] = ua_values[result_df[NON_PARALLEL]] - result_df.loc[result_df[NON_PARALLEL], UB] = ub_values[result_df[NON_PARALLEL]] - - return result_df - - -def check_line_intersections( - segments_df: DataFrame, - line_x1: float, - line_y1: float, - line_x2: float, - line_y2: float, - offset: RelativeOffsetCoordinate, -) -> DataFrame: - """ - Check if track segments intersect with a line. - - Args: - segments_df (DataFrame): DataFrame with track segments. - line_x1 (float): X-coordinate of the line's start point. - line_y1 (float): Y-coordinate of the line's start point. - line_x2 (float): X-coordinate of the line's end point. - line_y2 (float): Y-coordinate of the line's end point. - offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints - before intersection checks. - - Returns: - DataFrame: DataFrame with a single column INTERSECTS indicating whether each - segment intersects the line. - """ - if segments_df.empty: - return DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) - - # Bounding box of the line - line_min_x = min(line_x1, line_x2) - line_max_x = max(line_x1, line_x2) - line_min_y = min(line_y1, line_y2) - line_max_y = max(line_y1, line_y2) - - # Segment endpoints adjusted by offset (no copy/mutation) - sxa = segments_df[START_X] + segments_df[START_W] * offset.x - sya = segments_df[START_Y] + segments_df[START_H] * offset.y - exa = segments_df[END_X] + segments_df[END_W] * offset.x - eya = segments_df[END_Y] + segments_df[END_H] * offset.y - - # Segment bounding boxes - seg_min_x = sxa.where(sxa <= exa, exa) - seg_max_x = sxa.where(sxa >= exa, exa) - seg_min_y = sya.where(sya <= eya, eya) - seg_max_y = sya.where(sya >= eya, eya) - - # Candidate segments whose bbox intersects the line bbox - candidate_mask = ( - (seg_max_x >= line_min_x) - & (seg_min_x <= line_max_x) - & (seg_max_y >= line_min_y) - & (seg_min_y <= line_max_y) - ) - - # Prepare result DataFrame (all False by default) - intersects_df = DataFrame(index=segments_df.index, data=False, columns=[INTERSECTS]) - - if not candidate_mask.any(): - return intersects_df - - # Calculate intersection parameters for candidates only - candidate_df = segments_df.loc[candidate_mask] - params_df = calculate_intersection_parameters( - candidate_df, line_x1, line_y1, line_x2, line_y2, offset - ) - - # If all candidate segments are parallel or ua/ub couldn't be calculated - if ( - not params_df[NON_PARALLEL].any() - or params_df[UA].isna().all() - or params_df[UB].isna().all() - ): - return intersects_df - - # Create a mask for valid intersections - # (intersection point is on both line segments) - valid_intersection_mask = ( - (0 <= params_df[UA]) - & (params_df[UA] <= 1) - & (0 <= params_df[UB]) - & (params_df[UB] <= 1) - & params_df[NON_PARALLEL] - ) - - # Update the result DataFrame with True where intersections occur - if valid_intersection_mask.any(): - valid_idx = params_df.index[valid_intersection_mask] - intersects_df.loc[valid_idx, INTERSECTS] = True - - return intersects_df - - -def calculate_intersection_points( - segments_df: DataFrame, - line_x1: float, - line_y1: float, - line_x2: float, - line_y2: float, - offset: RelativeOffsetCoordinate, -) -> DataFrame: - """ - Calculate intersection points between track segments and a line. - - Args: - segments_df (DataFrame): DataFrame with track segments. - line_x1 (float): X-coordinate of the line's start point. - line_y1 (float): Y-coordinate of the line's start point. - line_x2 (float): X-coordinate of the line's end point. - line_y2 (float): Y-coordinate of the line's end point. - offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints - before intersection calculation. - - Returns: - DataFrame: DataFrame with columns INTERSECTION_X and INTERSECTION_Y containing - the coordinates of intersection points. - """ - if segments_df.empty: - return DataFrame( - index=segments_df.index, columns=[INTERSECTION_X, INTERSECTION_Y] - ) - - # Calculate intersection parameters (offset applied on-the-fly) - params_df = calculate_intersection_parameters( - segments_df, line_x1, line_y1, line_x2, line_y2, offset - ) - - # Initialize DataFrame for intersection coordinates - intersection_df = DataFrame( - index=segments_df.index, columns=[INTERSECTION_X, INTERSECTION_Y] - ) - - # If all segments are parallel or ua/ub couldn't be calculated - if ( - not params_df[NON_PARALLEL].any() - or params_df[UA].isna().all() - or params_df[UB].isna().all() - ): - return intersection_df - - # Create a mask for valid intersections - valid_intersection_mask = ( - (0 <= params_df[UA]) - & (params_df[UA] <= 1) - & (0 <= params_df[UB]) - & (params_df[UB] <= 1) - & params_df[NON_PARALLEL] - ) - - if valid_intersection_mask.any(): - # Adjusted segment endpoints with offset - sxa = segments_df[START_X] + segments_df[START_W] * offset.x - sya = segments_df[START_Y] + segments_df[START_H] * offset.y - exa = segments_df[END_X] + segments_df[END_W] * offset.x - eya = segments_df[END_Y] + segments_df[END_H] * offset.y - - # Calculate the intersection points for valid intersections - intersection_x = sxa + params_df[UA] * (exa - sxa) - intersection_y = sya + params_df[UA] * (eya - sya) - - # Update the result DataFrame with intersection coordinates - intersection_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_x[ - valid_intersection_mask - ] - intersection_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_y[ - valid_intersection_mask - ] - - return intersection_df - - -def find_line_intersections( - segments_df: DataFrame, - line_id: str, - start_x: float, - start_y: float, - end_x: float, - end_y: float, - offset: RelativeOffsetCoordinate | None = None, -) -> DataFrame: - """ - Find intersection points between track segments and a line. - - Args: - segments_df (DataFrame): DataFrame with track segments (output of - create_track_segments) - line_id (str): ID of the line - start_x (float): X-coordinate of the line's start point - start_y (float): Y-coordinate of the line's start point - end_x (float): X-coordinate of the line's end point - end_y (float): Y-coordinate of the line's end point - offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints - before intersection checks. - - Returns: - DataFrame: The input DataFrame with additional columns for intersection - information: - - INTERSECTS: Boolean indicating if the segment intersects with the - line - - INTERSECTION_X: X-coordinate of the intersection point (if any) - - INTERSECTION_Y: Y-coordinate of the intersection point (if any) - - INTERSECTION_LINE_ID: ID of the intersecting line - """ - if segments_df.empty: - return segments_df - - # Default offset if not provided - if offset is None: - offset = RelativeOffsetCoordinate(0.0, 0.0) - - # Use a shallow copy to avoid duplicating data blocks - result_df = segments_df.copy(deep=False) - - # Initialize new columns - result_df[INTERSECTS] = False - result_df[INTERSECTION_X] = None - result_df[INTERSECTION_Y] = None - result_df[INTERSECTION_LINE_ID] = None - - # Check which segments intersect with the line - intersects_df = check_line_intersections( - segments_df, start_x, start_y, end_x, end_y, offset - ) - - # If there are any intersections - if intersects_df[INTERSECTS].any(): - valid_intersection_mask = intersects_df[INTERSECTS] - - # Calculate intersection points only for intersecting segments - intersection_df = calculate_intersection_points( - segments_df.loc[valid_intersection_mask], - start_x, - start_y, - end_x, - end_y, - offset, - ) - - # Update the result DataFrame with intersection information - result_df.loc[valid_intersection_mask, INTERSECTS] = True - result_df.loc[valid_intersection_mask, INTERSECTION_X] = intersection_df[ - INTERSECTION_X - ] - result_df.loc[valid_intersection_mask, INTERSECTION_Y] = intersection_df[ - INTERSECTION_Y - ] - result_df.loc[valid_intersection_mask, INTERSECTION_LINE_ID] = line_id - - return result_df - - -def check_polygon_intersections( - segments_df: DataFrame, - polygon: Polygon, - offset: RelativeOffsetCoordinate, -) -> DataFrame: - """ - Check if track segments intersect with a polygon. - - A polygon consists of a set of lines. This function checks if each track segment - intersects with any of the lines in the polygon. - - Args: - segments_df (DataFrame): DataFrame with track segments (output of - create_track_segments). - polygon (Polygon): The polygon to check intersections with. - offset (RelativeOffsetCoordinate): Relative offset applied to segment endpoints - before intersection checks. - - Returns: - DataFrame: The input DataFrame with an additional column INTERSECTS_POLYGON - indicating if the segment intersects with the polygon. - """ - if segments_df.empty: - return segments_df - - # Shallow copy to add result column without duplicating data - result_df = segments_df.copy(deep=False) - - # Initialize the intersects-polygon column to False - result_df[INTERSECTS_POLYGON] = False - - # Get the coordinates of the polygon - polygon_coordinates = polygon.coordinates - - # Polygon bounding box - poly_min_x = min(c.x for c in polygon_coordinates) - poly_max_x = max(c.x for c in polygon_coordinates) - poly_min_y = min(c.y for c in polygon_coordinates) - poly_max_y = max(c.y for c in polygon_coordinates) - - # Segment endpoints adjusted by offset (no copy/mutation) - sxa = segments_df[START_X] + segments_df[START_W] * offset.x - sya = segments_df[START_Y] + segments_df[START_H] * offset.y - exa = segments_df[END_X] + segments_df[END_W] * offset.x - eya = segments_df[END_Y] + segments_df[END_H] * offset.y - - # Segment bounding boxes - seg_min_x = sxa.where(sxa <= exa, exa) - seg_max_x = sxa.where(sxa >= exa, exa) - seg_min_y = sya.where(sya <= eya, eya) - seg_max_y = sya.where(sya >= eya, eya) - - # Pre-filter segments whose bbox intersects polygon bbox - candidate_mask = ( - (seg_max_x >= poly_min_x) - & (seg_min_x <= poly_max_x) - & (seg_max_y >= poly_min_y) - & (seg_min_y <= poly_max_y) - ) - - if not candidate_mask.any(): - return result_df - - # Check intersections with each line segment of the polygon - for i in range(len(polygon_coordinates) - 1): - # Get the start and end points of the current line segment - start_x = polygon_coordinates[i].x - start_y = polygon_coordinates[i].y - end_x = polygon_coordinates[i + 1].x - end_y = polygon_coordinates[i + 1].y - - # Further prefilter by line bbox - line_min_x = min(start_x, end_x) - line_max_x = max(start_x, end_x) - line_min_y = min(start_y, end_y) - line_max_y = max(start_y, end_y) - cand2_mask = ( - (seg_max_x >= line_min_x) - & (seg_min_x <= line_max_x) - & (seg_max_y >= line_min_y) - & (seg_min_y <= line_max_y) - & candidate_mask - ) - if not cand2_mask.any(): - continue - - # Check which segments intersect with the current line segment - intersects_df = check_line_intersections( - segments_df.loc[cand2_mask], start_x, start_y, end_x, end_y, offset - ) - - # Update the result DataFrame using indices of True values - if intersects_df[INTERSECTS].any(): - true_idx = intersects_df.index[intersects_df[INTERSECTS]] - result_df.loc[true_idx, INTERSECTS_POLYGON] = True - - return result_df - - -def get_section_offset(section: Section) -> RelativeOffsetCoordinate: - if offset := section.relative_offset_coordinates.get(EventType.SECTION_ENTER): - return offset - raise ValueError(f"Section {section.id} has no offset") - - -class PandasTrackGeometryDataset(TrackGeometryDataset): - _segments_df: DataFrame - - def __init__( - self, offset: RelativeOffsetCoordinate, segments_df: Optional[DataFrame] = None - ): - """Initialize a PandasTrackGeometryDataset. - - Args: - segments_df (Optional[DataFrame], optional): DataFrame with track segments. - If None, an empty DataFrame will be created. Defaults to None. - """ - self._offset = offset - if segments_df is None: - self._segments_df = DataFrame() - else: - self._segments_df = segments_df - - # Ensure the DataFrame has the required columns - missing_columns = [ - col - for col in [ - TRACK_ID, - START_X, - START_Y, - START_W, - START_H, - END_X, - END_Y, - END_W, - END_H, - START_OCCURRENCE, - END_OCCURRENCE, - ] - if col not in self._segments_df.columns - ] - if not self._segments_df.empty and len(missing_columns) > 0: - raise ValueError( - "Segments DataFrame must have the required columns. " - f"Following columns are missing: {missing_columns}" - ) - - @property - def track_ids(self) -> set[str]: - """Get track ids of tracks stored in dataset. - - Returns: - set[str]: the track ids stored. - """ - if self._segments_df.empty: - return set() - return set(self._segments_df[TRACK_ID].unique()) - - @property - def offset(self) -> RelativeOffsetCoordinate: - return self._offset - - @property - def empty(self) -> bool: - """Check if the dataset is empty. - - Returns: - bool: True if the dataset is empty, False otherwise. - """ - return self._segments_df.empty - - @staticmethod - def from_track_dataset( - dataset: TrackDataset, offset: RelativeOffsetCoordinate - ) -> "TrackGeometryDataset": - """Create a PandasTrackGeometryDataset from a TrackDataset. - - Args: - dataset (TrackDataset): The source track dataset. Must be a - PandasTrackDataset. - offset (RelativeOffsetCoordinate): Relative offset to apply to track points - before segment creation. Currently not applied in this implementation. - - Returns: - TrackGeometryDataset: A dataset containing the generated track segments. - - Raises: - ValueError: If the dataset is not a PandasTrackDataset. - """ - if isinstance(dataset, PandasTrackDataset): - data = dataset.get_data() - segments = create_track_segments(data) - return PandasTrackGeometryDataset(offset, segments) - else: - raise ValueError( - "PandasTrackGeometryDataset can only be created from a " - "PandasTrackDataset" - ) - - def add_all(self, tracks: Iterable[Track]) -> "TrackGeometryDataset": - """Add tracks to existing dataset. - - Pre-existing tracks will be overwritten. - - Args: - tracks (Iterable[Track]): the tracks to add. - - Returns: - TrackGeometryDataset: the dataset with tracks added. - """ - # Convert tracks to DataFrame format - track_data = self._convert_tracks_to_dataframe(tracks) - if track_data.empty: - return self - - # Create segments from track data - new_segments = create_track_segments(track_data) - if new_segments.empty: - return self - - if self.empty: - # If current dataset is empty, return new dataset with the new segments - return PandasTrackGeometryDataset(self._offset, new_segments) - - # Merge with existing segments, overwriting duplicates - # Remove existing segments for tracks that are being added - new_track_ids = new_segments[TRACK_ID].unique() - existing_without_new = self._segments_df[ - ~self._segments_df[TRACK_ID].isin(new_track_ids) - ] - - # Combine existing (without overlaps) and new segments - combined_segments = DataFrame() - if not existing_without_new.empty and not new_segments.empty: - combined_segments = concat( - [existing_without_new, new_segments], ignore_index=True - ) - elif not existing_without_new.empty: - combined_segments = existing_without_new - elif not new_segments.empty: - combined_segments = new_segments - - return PandasTrackGeometryDataset(self._offset, combined_segments) - - def _convert_tracks_to_dataframe(self, tracks: Iterable[Track]) -> DataFrame: - """Convert tracks to DataFrame format. - - Args: - tracks (Iterable[Track]): tracks to convert. - - Returns: - DataFrame: tracks as dataframe with TRACK_ID, OCCURRENCE, X, Y, W, H. - """ - if not tracks: - return DataFrame() - - data = [] - for current in tracks: - track_id = current.id.id - for detection in current.detections: - data.append( - { - TRACK_ID: track_id, - OCCURRENCE: detection.occurrence, - X: detection.x, - Y: detection.y, - W: detection.w, - H: detection.h, - } - ) - - if not data: - return DataFrame() - - return DataFrame(data) - - def remove(self, ids: Sequence[str]) -> "TrackGeometryDataset": - """Remove track geometries with given ids from dataset. - - Args: - ids (Sequence[str]): the track geometries to remove. - - Returns: - TrackGeometryDataset: the dataset with tracks removed. - """ - if self.empty or not ids: - return self - - # Filter out segments with track IDs in the removal list - remaining_segments = self._segments_df[~self._segments_df[TRACK_ID].isin(ids)] - - return PandasTrackGeometryDataset(self._offset, remaining_segments) - - def get_for(self, track_ids: list[str]) -> "TrackGeometryDataset": - """Get geometries for given track ids if they exist. - - Ids that do not exist will not be included in the dataset. - - Args: - track_ids (list[str]): the track ids. - - Returns: - TrackGeometryDataset: the dataset with tracks. - """ - if self.empty or not track_ids: - return PandasTrackGeometryDataset(self._offset) - - # Filter segments to include only those with track IDs in the provided list - filtered_segments = self._segments_df[ - self._segments_df[TRACK_ID].isin(track_ids) - ] - - return PandasTrackGeometryDataset(self._offset, filtered_segments) - - def intersecting_tracks(self, sections: list[Section]) -> TrackIdSet: - """Return a set of tracks intersecting a set of sections. - - Args: - sections (list[Section]): the list of sections to intersect. - - Returns: - TrackIdSet: the track ids intersecting the given sections. - """ - if self.empty or not sections: - return PythonTrackIdSet() - - # Create a set to store the track IDs that intersect with any section - intersecting_track_ids: set[TrackId] = set() - - # Check intersections for each section - for section in sections: - # Get the coordinates of the section - coordinates = section.get_coordinates() - offset = get_section_offset(section) - - if ( - section.get_type() == SectionType.LINE - or section.get_type() == SectionType.CUTTING - ): - # For line sections, check if any track segment intersects with any leg - # of the line. A leg is formed by each consecutive pair of coordinates. - for i in range(len(coordinates) - 1): - start_x, start_y = coordinates[i].x, coordinates[i].y - end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y - - # Find intersections with this leg of the line - intersections = find_line_intersections( - self._segments_df, - section.id.serialize(), - start_x, - start_y, - end_x, - end_y, - offset, - ) - - # Add track IDs that intersect with the line to the result set - intersecting_segments = intersections[intersections[INTERSECTS]] - intersecting_track_ids.update( - TrackId(track_id) - for track_id in intersecting_segments[TRACK_ID].unique() - ) - elif section.get_type() == SectionType.AREA: - # For area sections, check if any track segment intersects with the - # polygon - polygon = Polygon(coordinates) - - # Check polygon intersections - intersections = check_polygon_intersections( - self._segments_df, polygon, offset - ) - - # Add track IDs that intersect with the polygon to the result set - intersecting_segments = intersections[intersections[INTERSECTS_POLYGON]] - intersecting_track_ids.update( - TrackId(track_id) - for track_id in intersecting_segments[TRACK_ID].unique() - ) - - return PythonTrackIdSet(intersecting_track_ids) - - def intersection_points( - self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - """Return the intersection points from tracks and the given sections. - - Args: - sections (list[Section]): the sections to intersect with. - - Returns: - dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - the intersection points. - """ - if self.empty or not sections: - return {} - - # Create a dictionary to store the intersection points for each track - result: dict[TrackId, list[tuple[SectionId, IntersectionPoint]]] = {} - - # Process only line sections - # (area sections don't have specific intersection points) - line_sections = [ - section - for section in sections - if section.get_type() == SectionType.LINE - or section.get_type() == SectionType.CUTTING - ] - - if not line_sections: - return {} - - # For each line section, find intersections with track segments - for section in line_sections: - # Get the coordinates of the section - coordinates = section.get_coordinates() - offset = get_section_offset(section) - - # Process each leg of the section (consecutive pair of coordinates) - for i in range(len(coordinates) - 1): - start_x, start_y = coordinates[i].x, coordinates[i].y - end_x, end_y = coordinates[i + 1].x, coordinates[i + 1].y - - # Find intersections with this leg of the line - intersections = find_line_intersections( - self._segments_df, - section.id.serialize(), - start_x, - start_y, - end_x, - end_y, - offset, - ) - - # Filter to only include segments that intersect with the line - intersecting_segments = intersections[intersections[INTERSECTS]] - - # Process each intersecting segment - for _, segment in intersecting_segments.iterrows(): - track_id = segment[TRACK_ID] - - # Calculate the relative position of the intersection point - # along the segment - segment_length_x = (segment[END_X] + segment[END_W] * offset.x) - ( - segment[START_X] + segment[START_W] * offset.x - ) - segment_length_y = (segment[END_Y] + segment[END_H] * offset.y) - ( - segment[START_Y] + segment[START_H] * offset.y - ) - segment_length = math.sqrt( - segment_length_x**2 + segment_length_y**2 - ) - # Avoid division by zero - if segment_length_x == 0 and segment_length_y == 0: - continue - - # Calculate the relative position (0 to 1) along the segment - intersection_length_x = segment[INTERSECTION_X] - ( - segment[START_X] + segment[START_W] * offset.x - ) - intersection_length_y = segment[INTERSECTION_Y] - ( - segment[START_Y] + segment[START_H] * offset.y - ) - intersection_length = math.sqrt( - intersection_length_x**2 + intersection_length_y**2 - ) - relative_position = intersection_length / segment_length - - # Create an IntersectionPoint - # The upper_index is 1 because we're dealing with segments - intersection_point = IntersectionPoint( - upper_index=1, relative_position=relative_position - ) - - # Add to the result dictionary - if track_id not in result: - result[track_id] = [] - result[track_id].append((section.id, intersection_point)) - - return result - - def contained_by_sections( - self, sections: list[Section] - ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - # TODO Do not implement - contains_result: dict[TrackId, list[tuple[SectionId, list[bool]]]] = ( - defaultdict(list) - ) - for _section in sections: - section_geom = area_section_to_shapely(_section) - - x_coords = self._segments_df["x"].to_numpy() - y_coords = self._segments_df["y"].to_numpy() - - contains_masks = contains(section_geom, x_coords, y_coords) - # contains_masks_old = self._segments_df[GEOMETRY].apply( - # lambda line: [ - # contains(section_geom, Point(p)) for p in get_coordinates(line) - # ] - # ) - tracks_contained = contains_masks[contains_masks.map(any)] - - if tracks_contained.empty: - continue - - tracks_contained.index = tracks_contained.index.map(TrackId) - for track_id, contains_mask in tracks_contained.to_dict().items(): - contains_result[track_id].append((_section.id, contains_mask)) - return contains_result - - def __eq__(self, other: Any) -> bool: - """Check if this dataset is equal to another dataset. - - Args: - other (Any): the other dataset to compare with. - - Returns: - bool: True if the datasets are equal, False otherwise. - """ - if not isinstance(other, PandasTrackGeometryDataset): - return False - - # If both are empty, they are equal - if self.empty and other.empty: - return True - - # If only one is empty, they are not equal - if self.empty != other.empty: - return False - - # Compare the DataFrames - # First, ensure both DataFrames have the same columns - if set(self._segments_df.columns) != set(other._segments_df.columns): - return False - - # Sort both DataFrames to ensure consistent comparison - self_df = self._segments_df.sort_values( - by=[TRACK_ID, START_OCCURRENCE] - ).reset_index(drop=True) - other_df = other._segments_df.sort_values( - by=[TRACK_ID, START_OCCURRENCE] - ).reset_index(drop=True) - - # Compare the DataFrames - return self_df.equals(other_df) - - -def area_section_to_shapely(section: Section) -> BaseGeometry: - geometry = shapely.Polygon([(c.x, c.y) for c in section.get_coordinates()]) - prepare(geometry) - return geometry diff --git a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py b/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py deleted file mode 100644 index 6031d057c..000000000 --- a/tests/OTAnalytics/plugin_datastore/track_geometry_store/test_geometry_store_comparison.py +++ /dev/null @@ -1,495 +0,0 @@ -""" -Test comparison between pandas and polars geometry store implementations. - -This module ensures that both pandas and polars implementations produce -identical results for all operations. -""" - -from datetime import datetime -from typing import Any, Union - -import polars as pl -import pytest -from pandas import DataFrame as PandasDataFrame -from pandas.testing import assert_frame_equal -from pytest import approx - -from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate -from OTAnalytics.domain.track import H, W -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - END_H, - END_OCCURRENCE, - END_W, - END_X, - END_Y, - INTERSECTION_X, - INTERSECTS, - OCCURRENCE, - START_H, - START_OCCURRENCE, - START_W, - START_X, - START_Y, - TRACK_ID, - PandasTrackGeometryDataset, -) -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - Polygon as PandasPolygon, -) -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import X, Y -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - check_polygon_intersections as pandas_check_polygon_intersections, -) -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - create_track_segments as pandas_create_track_segments, -) -from OTAnalytics.plugin_datastore.track_geometry_store.pandas_geometry_store import ( - find_line_intersections as pandas_find_line_intersections, -) -from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( - INTERSECTION_Y, - ROW_ID, - PolarsTrackGeometryDataset, -) -from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( - Polygon as PolarsPolygon, -) -from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( - check_polygon_intersections as polars_check_polygon_intersections, -) -from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( - create_track_segments as polars_create_track_segments, -) -from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( - find_line_intersections as polars_find_line_intersections, -) - - -# Test data fixtures -@pytest.fixture -def sample_track_data() -> dict[str, list[Any]]: - """Sample track data for testing.""" - return { - TRACK_ID: ["track1", "track1", "track2", "track2"], - OCCURRENCE: [ - datetime(2023, 1, 1, 10, 0, 0), - datetime(2023, 1, 1, 10, 0, 1), - datetime(2023, 1, 1, 10, 0, 0), - datetime(2023, 1, 1, 10, 0, 1), - ], - X: [0.0, 10.0, 20.0, 30.0], - Y: [0.0, 10.0, 20.0, 30.0], - W: [5.0, 5.0, 5.0, 5.0], - H: [5.0, 5.0, 5.0, 5.0], - } - - -@pytest.fixture -def sample_segments_data() -> dict[str, list[Any]]: - """Sample segments data for testing.""" - return { - TRACK_ID: ["track1", "track2"], - START_X: [60.0, 100.0], - START_Y: [0.0, 150.0], - END_X: [0.0, 110.0], - END_Y: [60.0, 160.0], - START_W: [12.0, 12.0], - START_H: [12.0, 12.0], - END_W: [12.0, 12.0], - END_H: [12.0, 12.0], - START_OCCURRENCE: [ - datetime(2023, 1, 1, 10, 0, 0), - datetime(2023, 1, 1, 10, 0, 0), - ], - END_OCCURRENCE: [ - datetime(2023, 1, 1, 10, 0, 1), - datetime(2023, 1, 1, 10, 0, 1), - ], - } - - -def dataframes_equal( - df1: Union[PandasDataFrame, pl.DataFrame], df2: Union[PandasDataFrame, pl.DataFrame] -) -> bool: - """Compare two dataframes for equality, handling pandas and polars.""" - # Convert both to pandas for comparison - if isinstance(df1, pl.DataFrame): - df1 = df1.to_pandas() - if isinstance(df2, pl.DataFrame): - df2 = df2.drop(ROW_ID, strict=False).to_pandas() - - # Handle empty dataframes - if df1.empty and df2.empty: - return True - if df1.empty or df2.empty: - return False - - # Sort by common columns for consistent comparison - sort_cols = [] - if TRACK_ID in df1.columns: - sort_cols.append(TRACK_ID) - if START_OCCURRENCE in df1.columns: - sort_cols.append(START_OCCURRENCE) - elif OCCURRENCE in df1.columns: - sort_cols.append(OCCURRENCE) - - if sort_cols: - df1 = df1.sort_values(sort_cols).reset_index(drop=True) - df2 = df2.sort_values(sort_cols).reset_index(drop=True) - - # Compare using pandas equals with appropriate tolerance for floats - try: - assert_frame_equal(df1, df2) - return True - except Exception: - try: - assert_frame_equal(df1.round(10), df2.round(10)) - return True - except Exception: - # Fallback to manual comparison - if set(df1.columns) != set(df2.columns): - return False - if len(df1) != len(df2): - return False - - for col in df1.columns: - if df1[col].dtype == "object" or df2[col].dtype == "object": - if not df1[col].equals(df2[col]): - return False - else: - try: - if not (abs(df1[col] - df2[col]) < 1e-10).all(): - return False - except Exception: - if not df1[col].equals(df2[col]): - return False - return True - - -def prepare_polars_data_frame( - data: dict[str, list[Any]], -) -> pl.DataFrame: - df = pl.DataFrame(data).with_row_index(ROW_ID) - if OCCURRENCE in df.columns: - return df.with_columns(pl.col(OCCURRENCE).dt.cast_time_unit("ns")) - else: - return df.with_columns( - pl.col(START_OCCURRENCE).dt.cast_time_unit("ns") - ).with_columns(pl.col(END_OCCURRENCE).dt.cast_time_unit("ns")) - - -def prepare_polars_for_assert( - df: pl.DataFrame, -) -> pl.DataFrame: - return df.drop(ROW_ID, strict=False) - - -def prepare_pandas_for_assert( - df: PandasDataFrame, -) -> PandasDataFrame: - for column in [INTERSECTION_X, INTERSECTION_Y]: - if column in df.columns: - df[column] = df[column].astype(float) - return df - - -class TestCreateTrackSegments: - """Test create_track_segments function for both implementations.""" - - def test_create_track_segments_empty_df(self) -> None: - """Test that empty DataFrames return empty results.""" - pandas_df = PandasDataFrame() - polars_df = pl.DataFrame() - - pandas_result = pandas_create_track_segments(pandas_df) - polars_result = polars_create_track_segments(polars_df) - - assert pandas_result.empty - assert polars_result.is_empty() - - def test_create_track_segments_single_track( - self, sample_track_data: dict[str, list[Any]] - ) -> None: - """Test segment creation with single track.""" - # Filter for single track - single_track_data = {k: [v[0], v[1]] for k, v in sample_track_data.items()} - - pandas_df = PandasDataFrame(single_track_data) - polars_df = prepare_polars_data_frame(single_track_data) - - pandas_result = pandas_create_track_segments(pandas_df) - polars_result = prepare_polars_for_assert( - polars_create_track_segments(polars_df) - ) - - # Both should produce one segment - assert len(pandas_result) == 1 - assert len(polars_result) == 1 - - # Results should be equivalent - assert dataframes_equal(pandas_result, polars_result) - - def test_create_track_segments_multiple_tracks( - self, sample_track_data: dict[str, list[Any]] - ) -> None: - """Test segment creation with multiple tracks.""" - pandas_df = PandasDataFrame(sample_track_data) - polars_df = prepare_polars_data_frame(sample_track_data) - - pandas_result = pandas_create_track_segments(pandas_df) - polars_result = polars_create_track_segments(polars_df) - - # Both should produce two segments (one per track) - assert len(pandas_result) == 2 - assert len(polars_result) == 2 - - # Results should be equivalent - assert dataframes_equal(pandas_result, polars_result) - - -class TestFindLineIntersections: - """Test find_line_intersections function for both implementations.""" - - def test_find_line_intersections_empty_df(self) -> None: - """Test with empty DataFrames.""" - pandas_df = PandasDataFrame() - polars_df = pl.DataFrame() - offset = RelativeOffsetCoordinate(0.0, 0.0) - - pandas_result = pandas_find_line_intersections( - pandas_df, "line1", 0.0, 0.0, 10.0, 10.0, offset - ) - polars_result = polars_find_line_intersections( - polars_df, "line1", 0.0, 0.0, 10.0, 10.0, offset - ) - - assert pandas_result.empty - assert polars_result.is_empty() - - def test_find_line_intersections_no_intersections( - self, sample_segments_data: dict[str, list[Any]] - ) -> None: - """Test with segments that don't intersect.""" - pandas_df = PandasDataFrame(sample_segments_data) - polars_df = prepare_polars_data_frame(sample_segments_data) - offset = RelativeOffsetCoordinate(0.0, 0.0) - - # Line that doesn't intersect - line_id = "non_intersecting_line" - start_x, start_y = 0.0, 100.0 - end_x, end_y = 5.0, 105.0 - - pandas_result = pandas_find_line_intersections( - pandas_df, line_id, start_x, start_y, end_x, end_y, offset - ) - polars_result = polars_find_line_intersections( - polars_df, line_id, start_x, start_y, end_x, end_y, offset - ) - - # Both should have no intersections - assert not pandas_result[INTERSECTS].any() - assert not polars_result[INTERSECTS].any() - - # Results should be equivalent - assert dataframes_equal( - prepare_pandas_for_assert(pandas_result), - prepare_polars_for_assert(polars_result), - ) - - @pytest.mark.parametrize( - "offset, expected_intersection", - [ - (RelativeOffsetCoordinate(0.0, 0.0), 30.0), - (RelativeOffsetCoordinate(0.5, 0.0), 33.0), - (RelativeOffsetCoordinate(1.0, 0.0), 36.0), - ], - ) - def test_find_line_intersections_with_intersections( - self, - sample_segments_data: dict[str, list[Any]], - offset: RelativeOffsetCoordinate, - expected_intersection: float, - ) -> None: - """Test with segments that intersect with the line.""" - pandas_df = PandasDataFrame(sample_segments_data) - polars_df = prepare_polars_data_frame(sample_segments_data) - - # Line that intersects with first segment - line_id = "intersecting_line" - start_x, start_y = 0.0, 0.0 - end_x, end_y = 120.0, 120.0 - - pandas_result = pandas_find_line_intersections( - pandas_df, line_id, start_x, start_y, end_x, end_y, offset - ) - polars_result = polars_find_line_intersections( - polars_df, line_id, start_x, start_y, end_x, end_y, offset - ) - - # Both should have same intersection pattern - assert dataframes_equal( - prepare_pandas_for_assert(pandas_result), - prepare_polars_for_assert(polars_result), - ) - - # Check specific intersection values - pandas_intersecting = pandas_result[pandas_result[INTERSECTS]] - polars_intersecting = polars_result.to_pandas()[ - polars_result.to_pandas()[INTERSECTS] - ] - - if len(pandas_intersecting) > 0: - assert pandas_intersecting.iloc[0][INTERSECTION_X] == approx( - expected_intersection - ) - assert polars_intersecting.iloc[0][INTERSECTION_X] == approx( - expected_intersection - ) - - -class TestCheckPolygonIntersections: - """Test check_polygon_intersections function for both implementations.""" - - def test_check_polygon_intersections_empty_df(self) -> None: - """Test with empty DataFrames.""" - pandas_df = PandasDataFrame() - polars_df = pl.DataFrame() - polygon_coords = [Coordinate(0, 0), Coordinate(10, 0), Coordinate(10, 10)] - pandas_polygon = PandasPolygon(polygon_coords) - polars_polygon = PolarsPolygon(polygon_coords) - offset = RelativeOffsetCoordinate(0.0, 0.0) - - pandas_result = pandas_check_polygon_intersections( - pandas_df, pandas_polygon, offset - ) - polars_result = polars_check_polygon_intersections( - polars_df, polars_polygon, offset - ) - - assert pandas_result.empty - assert polars_result.is_empty() - - def test_check_polygon_intersections_with_intersections( - self, sample_segments_data: dict[str, list[Any]] - ) -> None: - """Test polygon intersections.""" - pandas_df = PandasDataFrame(sample_segments_data) - polars_df = prepare_polars_data_frame(sample_segments_data) - - # Create a polygon that should intersect with some segments - polygon_coords = [ - Coordinate(0, 0), - Coordinate(70, 0), - Coordinate(70, 70), - Coordinate(0, 70), - ] - pandas_polygon = PandasPolygon(polygon_coords) - polars_polygon = PolarsPolygon(polygon_coords) - offset = RelativeOffsetCoordinate(0.0, 0.0) - - pandas_result = pandas_check_polygon_intersections( - pandas_df, pandas_polygon, offset - ) - polars_result = polars_check_polygon_intersections( - polars_df, polars_polygon, offset - ) - - # Results should be equivalent - assert dataframes_equal( - prepare_pandas_for_assert(pandas_result), - prepare_polars_for_assert(polars_result), - ) - - -class TestTrackGeometryDataset: - """Test TrackGeometryDataset implementations.""" - - def test_dataset_initialization(self) -> None: - """Test dataset initialization.""" - offset = RelativeOffsetCoordinate(0.0, 0.0) - - pandas_dataset = PandasTrackGeometryDataset(offset) - polars_dataset = PolarsTrackGeometryDataset(offset) - - assert pandas_dataset.empty - assert polars_dataset.empty - assert pandas_dataset.offset == polars_dataset.offset - - def test_dataset_track_ids( - self, sample_segments_data: dict[str, list[Any]] - ) -> None: - """Test track_ids property.""" - offset = RelativeOffsetCoordinate(0.0, 0.0) - pandas_df = PandasDataFrame(sample_segments_data) - polars_df = pl.DataFrame(sample_segments_data) - - pandas_dataset = PandasTrackGeometryDataset(offset, pandas_df) - polars_dataset = PolarsTrackGeometryDataset(offset, polars_df) - - assert pandas_dataset.track_ids == polars_dataset.track_ids - - def test_dataset_get_for(self, sample_segments_data: dict[str, list[Any]]) -> None: - """Test get_for method.""" - offset = RelativeOffsetCoordinate(0.0, 0.0) - pandas_df = PandasDataFrame(sample_segments_data) - polars_df = pl.DataFrame(sample_segments_data) - - pandas_dataset = PandasTrackGeometryDataset(offset, pandas_df) - polars_dataset = PolarsTrackGeometryDataset(offset, polars_df) - - track_ids = ["track1"] - pandas_result = pandas_dataset.get_for(track_ids) - polars_result = polars_dataset.get_for(track_ids) - - assert pandas_result.track_ids == polars_result.track_ids - # Both should contain only track1 - assert pandas_result.track_ids == {"track1"} - - def test_dataset_remove(self, sample_segments_data: dict[str, list[Any]]) -> None: - """Test remove method.""" - offset = RelativeOffsetCoordinate(0.0, 0.0) - pandas_df = PandasDataFrame(sample_segments_data) - polars_df = pl.DataFrame(sample_segments_data) - - pandas_dataset = PandasTrackGeometryDataset(offset, pandas_df) - polars_dataset = PolarsTrackGeometryDataset(offset, polars_df) - - ids_to_remove = ["track1"] - pandas_result = pandas_dataset.remove(ids_to_remove) - polars_result = polars_dataset.remove(ids_to_remove) - - assert pandas_result.track_ids == polars_result.track_ids - # Both should contain only track2 - assert pandas_result.track_ids == {"track2"} - - def test_dataset_equality_empty(self) -> None: - """Test dataset equality with empty datasets.""" - offset = RelativeOffsetCoordinate(0.0, 0.0) - - pandas_dataset1 = PandasTrackGeometryDataset(offset) - pandas_dataset2 = PandasTrackGeometryDataset(offset) - polars_dataset1 = PolarsTrackGeometryDataset(offset) - polars_dataset2 = PolarsTrackGeometryDataset(offset) - - assert pandas_dataset1 == pandas_dataset2 - assert polars_dataset1 == polars_dataset2 - - def test_dataset_equality_with_data( - self, sample_segments_data: dict[str, list[Any]] - ) -> None: - """Test dataset equality with data.""" - offset = RelativeOffsetCoordinate(0.0, 0.0) - pandas_df = PandasDataFrame(sample_segments_data) - polars_df = pl.DataFrame(sample_segments_data) - - pandas_dataset1 = PandasTrackGeometryDataset(offset, pandas_df.copy()) - pandas_dataset2 = PandasTrackGeometryDataset(offset, pandas_df.copy()) - polars_dataset1 = PolarsTrackGeometryDataset(offset, polars_df.clone()) - polars_dataset2 = PolarsTrackGeometryDataset(offset, polars_df.clone()) - - assert pandas_dataset1 == pandas_dataset2 - assert polars_dataset1 == polars_dataset2 - - -if __name__ == "__main__": - pytest.main([__file__]) From 5282b87b76f3aaff20eee9afe5422211ad27f128 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 15:21:46 +0200 Subject: [PATCH 075/128] Remove clean architecture violation by introducing a TrackIdSetFactory --- .../use_cases/highlight_intersections.py | 57 ++++++++++++------- .../domain/track_dataset/track_dataset.py | 16 ++++++ .../plugin_datastore/polars_track_id_set.py | 12 +++- OTAnalytics/plugin_ui/base_application.py | 12 +++- .../plugin_ui/visualization/visualization.py | 13 +++++ .../use_cases/test_highlight_intersections.py | 47 ++++++++++----- tests/OTAnalytics/plugin_ui/test_cli.py | 7 ++- 7 files changed, 126 insertions(+), 38 deletions(-) diff --git a/OTAnalytics/application/use_cases/highlight_intersections.py b/OTAnalytics/application/use_cases/highlight_intersections.py index 36584eb14..349db8c7a 100644 --- a/OTAnalytics/application/use_cases/highlight_intersections.py +++ b/OTAnalytics/application/use_cases/highlight_intersections.py @@ -14,10 +14,13 @@ from OTAnalytics.domain.flow import FlowId, FlowRepository from OTAnalytics.domain.section import SectionId from OTAnalytics.domain.track import Track, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import ( + TrackIdSet, + TrackIdSetFactory, + concat, +) from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class IntersectionRepository(ABC): @@ -170,13 +173,13 @@ def get_ids(self) -> TrackIdSet: def _calculate_new_intersections( self, section_ids_to_process: set[SectionId] ) -> TrackIdSet: - new_intersections: set[TrackId] = set() + new_intersections: list[TrackIdSet] = [] if section_ids_to_process: sections = self._get_section_by_id(section_ids_to_process) intersections = self._tracks_intersecting_sections(sections) self._intersection_repository.store(intersections) - new_intersections.update(*intersections.values()) - return PythonTrackIdSet(new_intersections) + new_intersections.extend(intersections.values()) + return concat(new_intersections) class TracksNotIntersectingSelection(TrackIdProvider): @@ -217,11 +220,13 @@ def __init__( event_repository: EventRepository, flow_repository: FlowRepository, flow_state: FlowState, + track_id_set_factory: TrackIdSetFactory, ) -> None: self._assigner = assigner self._event_repository = event_repository self._flow_repository = flow_repository self._flow_state = flow_state + self._factory = track_id_set_factory def get_ids(self) -> TrackIdSet: events = self._event_repository.get_all() @@ -233,9 +238,9 @@ def get_ids(self) -> TrackIdSet: ids = set() for assignment in assignments: if assignment.assignment.id in self._flow_state.selected_flows.get(): - ids.add(TrackId(assignment.road_user)) + ids.add(assignment.road_user) print(f"Tracks assigned to selected flow(s): {len(ids)}") - return PythonTrackIdSet(ids) + return self._factory.create(ids) class TracksAssignedToAllFlows(TrackIdProvider): @@ -252,15 +257,21 @@ def __init__( assigner: RoadUserAssigner, event_repository: EventRepository, flow_repository: FlowRepository, + track_id_set_factory: TrackIdSetFactory, ) -> None: self._assigner = assigner self._event_repository = event_repository self._flow_repository = flow_repository + self._factory = track_id_set_factory def get_ids(self) -> TrackIdSet: all_flow_ids = [flow.id for flow in self._flow_repository.get_all()] return TracksAssignedToGivenFlows( - self._assigner, self._event_repository, self._flow_repository, all_flow_ids + self._assigner, + self._event_repository, + self._flow_repository, + all_flow_ids, + self._factory, ).get_ids() @@ -280,11 +291,13 @@ def __init__( event_repository: EventRepository, flow_repository: FlowRepository, flow_ids: list[FlowId], + track_id_set_factory: TrackIdSetFactory, ) -> None: self._assigner = assigner self._event_repository = event_repository self._flow_repository = flow_repository self._flows = list(flow_ids) + self._factory = track_id_set_factory def get_ids(self) -> TrackIdSet: events = self._event_repository.get_all() @@ -296,8 +309,8 @@ def get_ids(self) -> TrackIdSet: ids = set() for assignment in assignments: if assignment.assignment.id in self._flows: - ids.add(TrackId(assignment.road_user)) - return PythonTrackIdSet(ids) + ids.add(assignment.road_user) + return self._factory.create(ids) class TracksOverlapOccurrenceWindow(TrackIdProvider): @@ -305,6 +318,7 @@ def __init__( self, track_repository: TrackRepository, track_view_state: TrackViewState, + track_id_set_factory: TrackIdSetFactory, other: Optional[TrackIdProvider] = None, ) -> None: """Returns track ids that overlap with the current date range filter. @@ -319,16 +333,17 @@ def __init__( self._other = other self._track_repository = track_repository self._track_view_state = track_view_state + self._factory = track_id_set_factory def get_ids(self) -> TrackIdSet: if self._other: - tracks = self._get_other_track_ids() + tracks = self._get_other_track() else: tracks = self._track_repository.get_all() - return PythonTrackIdSet(self._filter(tracks)) + return self._factory.create(self._filter(tracks)) - def _get_other_track_ids(self) -> Iterable[Track]: + def _get_other_track(self) -> Iterable[Track]: if self._other: track_ids = self._other.get_ids() tracks: list[Track] = [] @@ -338,14 +353,14 @@ def _get_other_track_ids(self) -> Iterable[Track]: return tracks return [] - def _filter(self, tracks: Iterable[Track]) -> Iterable[TrackId]: + def _filter(self, tracks: Iterable[Track]) -> set[TrackId]: date_range = self._track_view_state.filter_element.get().date_range start_date_filter = date_range.start_date end_date_filter = date_range.end_date match (start_date_filter, end_date_filter): case (datetime() as start_date_filter, datetime() as end_date_filter): - return [ + return { track.id for track in tracks if self._has_overlap( @@ -354,9 +369,9 @@ def _filter(self, tracks: Iterable[Track]) -> Iterable[TrackId]: track.start, track.end, ) - ] + } case (None, datetime() as end_date_filter): - return [ + return { track.id for track in tracks if self._has_overlap( @@ -365,9 +380,9 @@ def _filter(self, tracks: Iterable[Track]) -> Iterable[TrackId]: track.start, track.end, ) - ] + } case (datetime() as start_date_filter, None): - return [ + return { track.id for track in tracks if self._has_overlap( @@ -376,9 +391,9 @@ def _filter(self, tracks: Iterable[Track]) -> Iterable[TrackId]: track.start, track.end, ) - ] + } case _: - return [track.id for track in tracks] + return {track.id for track in tracks} @staticmethod def _has_overlap( diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 5dfc9c6c9..55dc6186a 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -110,6 +110,12 @@ def difference(self, other: "TrackIdSet") -> "TrackIdSet": raise NotImplementedError +class TrackIdSetFactory(ABC): + @abstractmethod + def create(self, track_ids: Iterable[TrackId] | Iterable[str]) -> TrackIdSet: + raise NotImplementedError + + class EmptyTrackIdSet(TrackIdSet): def __eq__(self, other: Any) -> bool: return isinstance(other, EmptyTrackIdSet) @@ -130,6 +136,16 @@ def difference(self, other: "TrackIdSet") -> "TrackIdSet": return self +def concat( + track_id_sets: Iterable[TrackIdSet], +) -> TrackIdSet: + """Concatenate multiple track ID sets into a single track ID set.""" + result: TrackIdSet = EmptyTrackIdSet() + for current in track_id_sets: + result = result.union(current) + return result + + class TrackDataset(ABC): @property @abstractmethod diff --git a/OTAnalytics/plugin_datastore/polars_track_id_set.py b/OTAnalytics/plugin_datastore/polars_track_id_set.py index bf762d865..e4accd17e 100644 --- a/OTAnalytics/plugin_datastore/polars_track_id_set.py +++ b/OTAnalytics/plugin_datastore/polars_track_id_set.py @@ -3,7 +3,7 @@ import polars as pl from OTAnalytics.domain.track import TrackId, unpack -from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet, TrackIdSetFactory class PolarsTrackIdSet(TrackIdSet): @@ -11,7 +11,9 @@ class PolarsTrackIdSet(TrackIdSet): Polars-based implementation of TrackIdSet using a Series for optimal performance. """ - def __init__(self, track_ids: Iterable[TrackId] | pl.Series | None = None): + def __init__( + self, track_ids: Iterable[TrackId] | Iterable[str] | pl.Series | None = None + ): if track_ids is None: self._series = pl.Series([], dtype=pl.String) elif isinstance(track_ids, pl.Series): @@ -72,3 +74,9 @@ def difference(self, other: "TrackIdSet") -> "TrackIdSet": ~self._series.is_in(list(other_strings)) ) return PolarsTrackIdSet(difference_series) + + +class PolarsTrackIdSetFactory(TrackIdSetFactory): + + def create(self, track_ids: Iterable[TrackId] | Iterable[str]) -> TrackIdSet: + return PolarsTrackIdSet(track_ids) diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index fba393fef..35352fcc7 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -183,9 +183,11 @@ from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.remark import RemarkRepository from OTAnalytics.domain.section import SectionRepository +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSetFactory from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository +from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSetFactory from OTAnalytics.plugin_datastore.polars_track_store import ( POLARS_TRACK_GEOMETRY_FACTORY, PolarsByMaxConfidence, @@ -675,6 +677,7 @@ def visualization_builder( self.color_palette_provider, self.progressbar_builder, self.track_image_factory, + self.track_id_set_factory, ) @cached_property @@ -903,7 +906,10 @@ def calculate_track_statistics(self) -> CalculateTrackStatistics: ) tracks_assigned_to_all_flows = FilteredTrackIdProviderByTrackIdProvider( TracksAssignedToAllFlows( - self.road_user_assigner, self.event_repository, self.flow_repository + self.road_user_assigner, + self.event_repository, + self.flow_repository, + self.track_id_set_factory, ), self.all_filtered_track_ids, ) @@ -985,6 +991,10 @@ def polars_by_max_confidence(self) -> PolarsByMaxConfidence: def track_geometry_factory(self) -> POLARS_TRACK_GEOMETRY_FACTORY: return PolarsTrackGeometryDataset.from_track_dataset + @cached_property + def track_id_set_factory(self) -> TrackIdSetFactory: + return PolarsTrackIdSetFactory() + @cached_property def remove_events_by_road_user_id(self) -> RemoveEventsByRoadUserId: return RemoveEventsByRoadUserId(self.event_repository) diff --git a/OTAnalytics/plugin_ui/visualization/visualization.py b/OTAnalytics/plugin_ui/visualization/visualization.py index 349e264c9..6a1fb9ff2 100644 --- a/OTAnalytics/plugin_ui/visualization/visualization.py +++ b/OTAnalytics/plugin_ui/visualization/visualization.py @@ -40,6 +40,7 @@ from OTAnalytics.domain.flow import FlowId, FlowRepository from OTAnalytics.domain.progress import ProgressbarBuilder from OTAnalytics.domain.section import SectionId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSetFactory from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_filter.dataframe_filter import DataFrameFilterBuilder @@ -145,6 +146,7 @@ def __init__( flow_repository: FlowRepository, state: FlowState, track_repository: TrackRepository, + track_id_set_factory: TrackIdSetFactory, ) -> None: self._other = other self._event_repository = event_repository @@ -152,6 +154,7 @@ def __init__( self._assigner = assigner self._state = state self._track_repository = track_repository + self._track_id_set_factory = track_id_set_factory def get_data(self) -> DataFrame: return FilterById( @@ -162,6 +165,7 @@ def get_data(self) -> DataFrame: self._event_repository, self._flow_repository, self._state.selected_flows.get(), + track_id_set_factory=self._track_id_set_factory, ), self._track_repository, ), @@ -199,6 +203,7 @@ def __init__( color_palette_provider: ColorPaletteProvider, pulling_progressbar_builder: ProgressbarBuilder, track_image_factory: TrackImageFactory, + track_id_set_factory: TrackIdSetFactory, enable_single_legend: bool = True, enable_multi_legend: bool = False, ) -> None: @@ -208,6 +213,7 @@ def __init__( self._color_palette_provider = color_palette_provider self._pulling_progressbar_builder = pulling_progressbar_builder self._track_image_factory = track_image_factory + self._track_id_set_factory = track_id_set_factory self._enable_single_legend = enable_single_legend self._enable_multi_legend = enable_multi_legend self._track_repository = datastore._track_repository @@ -512,6 +518,7 @@ def _create_highlight_start_end_point_not_assigned_to_flows_plotter( self._flow_repository, flow_state, self._track_repository, + track_id_set_factory=self._track_id_set_factory, ) cached_plotter = CachedPlotter( self._create_track_start_end_point_plotter( @@ -579,6 +586,7 @@ def _create_highlight_tracks_not_assigned_to_flows_plotter( self._flow_repository, flow_state, self._track_repository, + track_id_set_factory=self._track_id_set_factory, ) cached_plotter = CachedPlotter( self._create_track_geometry_plotter( @@ -757,6 +765,7 @@ def _create_track_start_end_point_data_provider( other=id_filter, track_repository=self._track_repository, track_view_state=self._track_view_state, + track_id_set_factory=self._track_id_set_factory, ), ) @@ -838,6 +847,7 @@ def _create_tracks_start_end_point_intersecting_given_sections_filter( ), track_repository=self._track_repository, track_view_state=self._track_view_state, + track_id_set_factory=self._track_id_set_factory, ), ) @@ -892,6 +902,7 @@ def _create_tracks_start_end_point_not_intersecting_given_sections_filter( ), track_repository=self._track_repository, track_view_state=self._track_view_state, + track_id_set_factory=self._track_id_set_factory, ), ) @@ -905,6 +916,7 @@ def _create_tracks_assigned_to_flows_filter( self._event_repository, self._flow_repository, [flow], + track_id_set_factory=self._track_id_set_factory, ), ) @@ -950,6 +962,7 @@ def _create_tracks_not_assigned_to_flows_filter( self._event_repository, self._flow_repository, [flow], + track_id_set_factory=self._track_id_set_factory, ), self._track_repository, ), diff --git a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py index c19c8ea22..e698d15f3 100644 --- a/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py +++ b/tests/OTAnalytics/application/use_cases/test_highlight_intersections.py @@ -35,7 +35,11 @@ from OTAnalytics.domain.flow import Flow, FlowId, FlowRepository from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Detection, Track, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackIdSet +from OTAnalytics.domain.track_dataset.track_dataset import ( + EmptyTrackIdSet, + TrackIdSet, + TrackIdSetFactory, +) from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @@ -158,8 +162,8 @@ def test_get_ids( intersection_repository: Mock, ) -> None: sections = [section_1, section_2] - section_1_tracks = {track_id_1} - section_2_tracks = {track_id_2} + section_1_tracks = PythonTrackIdSet({track_id_1}) + section_2_tracks = PythonTrackIdSet({track_id_2}) original_track_ids = { section_1.id: section_1_tracks, section_2.id: section_2_tracks, @@ -354,17 +358,22 @@ def test_get_ids(self) -> None: flow_repository = Mock(spec=FlowRepository) flow_repository.get_all.return_value = [first_flow, second_flow] + track_id_set = Mock(spec=TrackIdSet) + mock_factory = Mock(spec=TrackIdSetFactory) + mock_factory.create.return_value = track_id_set + tracks_assigned_to_flow = TracksAssignedToSelectedFlows( - assigner, event_repository, flow_repository, flow_state + assigner, event_repository, flow_repository, flow_state, mock_factory ) - track_ids = list(tracks_assigned_to_flow.get_ids()) + track_ids = tracks_assigned_to_flow.get_ids() - assert track_ids == [TrackId("1")] + assert track_ids == track_id_set event_repository.get_all.assert_called_once() flow_repository.get_all.assert_called_once() assert selected_flows.get.call_count == 2 assigner.assign.assert_called_once_with([event], [first_flow, second_flow]) assignments.as_list.assert_called_once() + mock_factory.create.assert_called_once_with({"1"}) class TestTracksOverlapOccurrenceWindow: @@ -452,24 +461,31 @@ def test_has_overlap_wrong_order_of_arguments(self) -> None: def test_get_ids(self) -> None: track_repository = Mock(spec=TrackRepository) track_view_state = Mock(spec=TrackViewState) + track_id_set = Mock(spec=TrackIdSet) + mock_factory = Mock(spec=TrackIdSetFactory) + mock_factory.create.return_value = track_id_set first_id = Mock(spec=TrackId) tracks = [Mock(spec=Track), None] track_repository.get_all.return_value = tracks with patch.object( - TracksOverlapOccurrenceWindow, "_filter", return_value=[first_id] + TracksOverlapOccurrenceWindow, "_filter", return_value={first_id} ): id_provider = TracksOverlapOccurrenceWindow( - track_repository, track_view_state + track_repository, track_view_state, mock_factory ) result_ids = id_provider.get_ids() - assert result_ids == PythonTrackIdSet([first_id]) + assert result_ids == track_id_set track_repository.get_all.assert_called_once() + mock_factory.create.assert_called_once_with({first_id}) def test_get_ids_as_decorator(self) -> None: track_repository = Mock(spec=TrackRepository) track_view_state = Mock(spec=TrackViewState) + track_id_set = Mock(spec=TrackIdSet) + mock_factory = Mock(spec=TrackIdSetFactory) + mock_factory.create.return_value = track_id_set first_id = Mock(spec=TrackId) track_ids = [first_id, Mock(spec=TrackId)] tracks = [Mock(spec=Track), None] @@ -478,19 +494,21 @@ def test_get_ids_as_decorator(self) -> None: other.get_ids.return_value = track_ids with patch.object( - TracksOverlapOccurrenceWindow, "_filter", return_value=[first_id] + TracksOverlapOccurrenceWindow, "_filter", return_value={first_id} ): id_provider = TracksOverlapOccurrenceWindow( track_repository, track_view_state, other=other, + track_id_set_factory=mock_factory, ) result_ids = id_provider.get_ids() - assert result_ids == PythonTrackIdSet([first_id]) + assert result_ids == track_id_set assert track_repository.get_for.call_args_list == [ call(id) for id in track_ids ] + mock_factory.create.assert_called_once_with({first_id}) @pytest.mark.parametrize( ( @@ -526,6 +544,8 @@ def test_filter( observable_property.get.return_value = filter_element track_view_state.filter_element = observable_property + mock_factory = Mock(spec=TrackIdSetFactory) + start_detection = Mock(spec=Detection) start_time = datetime(2020, 1, 1, 13) start_detection.occurrence = start_time @@ -556,11 +576,11 @@ def test_filter( track.end.return_value = end_detection id_provider = TracksOverlapOccurrenceWindow( - track_repository, track_view_state + track_repository, track_view_state, mock_factory ) result_ids = id_provider._filter([track]) - assert result_ids == [track_id] + assert result_ids == {track_id} mock_start.assert_called_once() mock_end.assert_called_once() if filter_start or filter_end: @@ -572,3 +592,4 @@ def test_filter( ) else: mock_has_overlap.assert_not_called() + mock_factory.create.assert_not_called() diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index 0bb1ca526..474a5faa3 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -91,6 +91,7 @@ from OTAnalytics.domain.section import SectionId, SectionRepository, SectionType from OTAnalytics.domain.track import TrackId from OTAnalytics.domain.track_repository import TrackRepository +from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSetFactory from OTAnalytics.plugin_datastore.python_track_store import ( ByMaxConfidence, PythonTrackDataset, @@ -549,8 +550,12 @@ def cli_dependencies(self) -> dict[str, Any]: ) get_all_tracks = GetAllTracks(track_repository) get_cutting_sections = GetCuttingSections(section_repository) + track_id_set_factory = PolarsTrackIdSetFactory() tracks_assigned_to_all_flows = TracksAssignedToAllFlows( - SimpleRoadUserAssigner(), event_repository, flow_repository + SimpleRoadUserAssigner(), + event_repository, + flow_repository, + track_id_set_factory, ) export_track_statistics = ExportTrackStatistics( CalculateTrackStatistics( From c8f324cab404f38e20204837d3053f7bacad2727 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 15:24:31 +0200 Subject: [PATCH 076/128] Increase chunk size to speed up regression tests --- tests/regression_otanalytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/regression_otanalytics.py b/tests/regression_otanalytics.py index 126331063..a06b2c547 100644 --- a/tests/regression_otanalytics.py +++ b/tests/regression_otanalytics.py @@ -48,7 +48,7 @@ def otflow_file(test_data_dir: Path) -> str: @pytest.fixture(scope="module") def cli_chunk_size() -> int: - return 5 + return 1000 @pytest.fixture From 8cefbf21b348e305af6d051349cab8d110c7d332 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 15:25:26 +0200 Subject: [PATCH 077/128] Fix clean architecture violation --- OTAnalytics/application/use_cases/inside_cutting_section.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/OTAnalytics/application/use_cases/inside_cutting_section.py b/OTAnalytics/application/use_cases/inside_cutting_section.py index 2421d8f83..1e8a24bb2 100644 --- a/OTAnalytics/application/use_cases/inside_cutting_section.py +++ b/OTAnalytics/application/use_cases/inside_cutting_section.py @@ -4,7 +4,6 @@ from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackIdSet from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackListObserver, TrackRepositoryEvent -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class TrackIdsInsideCuttingSections(TrackIdProvider): @@ -18,7 +17,7 @@ def get_ids(self) -> TrackIdSet: track_dataset = self._get_tracks.as_dataset() cutting_sections = self._get_cutting_sections() if not cutting_sections: - return PythonTrackIdSet() + return EmptyTrackIdSet() return track_dataset.ids_inside(cutting_sections) From 9a4d8cd3257ac994a76a2d37351a9efb530327ed Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 15:45:03 +0200 Subject: [PATCH 078/128] Fix clean architecture violation by using a factory --- .../application/analysis/traffic_counting.py | 21 +++-- .../use_cases/road_user_assignment_export.py | 7 +- OTAnalytics/plugin_ui/base_application.py | 2 +- .../analysis/test_traffic_counting.py | 87 +++++++++++++------ .../test_road_user_assignment_export.py | 3 +- tests/OTAnalytics/plugin_ui/test_cli.py | 13 ++- 6 files changed, 90 insertions(+), 43 deletions(-) diff --git a/OTAnalytics/application/analysis/traffic_counting.py b/OTAnalytics/application/analysis/traffic_counting.py index 9ab521cec..4e2be684c 100644 --- a/OTAnalytics/application/analysis/traffic_counting.py +++ b/OTAnalytics/application/analysis/traffic_counting.py @@ -27,6 +27,7 @@ ) from OTAnalytics.domain.flow import Flow, FlowRepository from OTAnalytics.domain.section import Section, SectionId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet, TrackIdSetFactory from OTAnalytics.domain.types import EventType DATETIME_FORMAT = "%Y-%m-%d_%H-%M-%S" @@ -603,16 +604,23 @@ class RoadUserAssignments: """ @property - def road_user_ids(self) -> list[str]: + def road_user_ids(self) -> TrackIdSet: """Returns a sorted list of all road user ids within this group of assignments. Returns: list[str]: the road user ids. """ - return sorted([assignment.road_user for assignment in self._assignments]) + return self._track_id_set_factory.create( + {assignment.road_user for assignment in self._assignments} + ) - def __init__(self, assignments: list[RoadUserAssignment]) -> None: + def __init__( + self, + assignments: list[RoadUserAssignment], + track_id_set_factory: TrackIdSetFactory, + ) -> None: self._assignments = assignments.copy() + self._track_id_set_factory = track_id_set_factory def tag(self, by: Tagger) -> TaggedAssignments: """ @@ -705,7 +713,9 @@ class SimpleRoadUserAssigner(RoadUserAssigner): """ def __init__( - self, flow_selection: FlowSelection = MaxDurationFlowSelection() + self, + track_id_set_factory: TrackIdSetFactory, + flow_selection: FlowSelection = MaxDurationFlowSelection(), ) -> None: """ Initialize the SimpleRoadUserAssigner with a flow selection strategy. @@ -715,6 +725,7 @@ def __init__( Defaults to MaxDurationFlowSelection. """ self._flow_selection = flow_selection + self._track_id_set_factory = track_id_set_factory def assign(self, events: Iterable[Event], flows: list[Flow]) -> RoadUserAssignments: """ @@ -800,7 +811,7 @@ def __assign_user_to_flow( road_user_id=road_user_id, road_user_type=road_user_type ) assignments.extend(user_assignments) - return RoadUserAssignments(assignments) + return RoadUserAssignments(assignments, self._track_id_set_factory) def __create_candidates( self, diff --git a/OTAnalytics/application/use_cases/road_user_assignment_export.py b/OTAnalytics/application/use_cases/road_user_assignment_export.py index 79e989c0b..f11fdfc77 100644 --- a/OTAnalytics/application/use_cases/road_user_assignment_export.py +++ b/OTAnalytics/application/use_cases/road_user_assignment_export.py @@ -16,8 +16,6 @@ from OTAnalytics.domain.event import EventRepository from OTAnalytics.domain.flow import FlowRepository from OTAnalytics.domain.section import Section, SectionId, SectionRepository -from OTAnalytics.domain.track import TrackId -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet MaxConfidenceLookupTable = dict[str, float] MaxConfidenceProvider = Callable[[list[str]], MaxConfidenceLookupTable] @@ -198,10 +196,9 @@ def _convert(self, assignments: RoadUserAssignments) -> list[dict]: def _get_max_conf_lookup_table_for( self, assignments: RoadUserAssignments ) -> MaxConfidenceLookupTable: - track_id_set = PythonTrackIdSet( - [TrackId(track_id) for track_id in assignments.road_user_ids] + return self._get_all_tracks.as_dataset().get_max_confidences_for( + assignments.road_user_ids ) - return self._get_all_tracks.as_dataset().get_max_confidences_for(track_id_set) def _get_section_by_id(self, section_id: SectionId) -> Section: result = self._section_repository.get(section_id) diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index 35352fcc7..c6c7fc76d 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -548,7 +548,7 @@ def road_user_assigner(self) -> RoadUserAssigner: @cached_property def simple_road_user_assigner(self) -> RoadUserAssigner: - return SimpleRoadUserAssigner() + return SimpleRoadUserAssigner(self.track_id_set_factory) @cached_property def file_state(self) -> FileState: diff --git a/tests/OTAnalytics/application/analysis/test_traffic_counting.py b/tests/OTAnalytics/application/analysis/test_traffic_counting.py index ddad637a9..2ef7d0ced 100644 --- a/tests/OTAnalytics/application/analysis/test_traffic_counting.py +++ b/tests/OTAnalytics/application/analysis/test_traffic_counting.py @@ -55,6 +55,7 @@ from OTAnalytics.domain.geometry import DirectionVector2D, ImageCoordinate from OTAnalytics.domain.section import SectionId from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet from OTAnalytics.domain.types import EventType from tests.utils.builders.track_builder import TrackBuilder @@ -70,6 +71,10 @@ def track(track_builder: TrackBuilder) -> Track: return track_builder.build_track() +def mock_factory() -> Mock: + return Mock() + + @pytest.mark.parametrize( "start_time,expected_start_time,expected_end_time", [ @@ -320,38 +325,39 @@ def __init__(self) -> None: def build_assignment_test_cases( self, + mock_factory: Mock, ) -> list[tuple[list[Event], list[Flow], RoadUserAssignments]]: return [ - self.__create_complex_test_case(), - self.__create_single_track_multiple_selection(), - self.__create_single_track_single_event(), - self.__create_tracks_without_match(), - self.__create_unordered_events(), - self.__create_same_frame_events(), + self.__create_complex_test_case(mock_factory), + self.__create_single_track_multiple_selection(mock_factory), + self.__create_single_track_single_event(mock_factory), + self.__create_tracks_without_match(mock_factory), + self.__create_unordered_events(mock_factory), + self.__create_same_frame_events(mock_factory), ] def __create_single_track_single_event( - self, + self, mock_factory: Mock ) -> tuple[list[Event], list[Flow], RoadUserAssignments]: events = [ create_event(self.first_track, self.south_section_id, 0), ] - expected_result: RoadUserAssignments = RoadUserAssignments([]) + expected_result: RoadUserAssignments = RoadUserAssignments([], mock_factory) return (events, self.flows, expected_result) def __create_tracks_without_match( - self, + self, mock_factory: Mock ) -> tuple[list[Event], list[Flow], RoadUserAssignments]: events = [ create_event(self.first_track, self.west_section_id, 0), create_event(self.first_track, self.south_section_id, 1), ] - expected_result = RoadUserAssignments([]) + expected_result = RoadUserAssignments([], mock_factory) return (events, self.flows, expected_result) def __create_unordered_events( - self, + self, mock_factory: Mock ) -> tuple[list[Event], list[Flow], RoadUserAssignments]: first_south = create_event(self.first_track, self.south_section_id, 1) first_north = create_event(self.first_track, self.north_section_id, 0) @@ -367,13 +373,14 @@ def __create_unordered_events( self.north_to_south, EventPair(first_north, first_south), ) - ] + ], + mock_factory, ) return (events, self.flows, expected_result) def __create_single_track_multiple_selection( - self, + self, mock_factory: Mock ) -> tuple[list[Event], list[Flow], RoadUserAssignments]: first_south = create_event(self.first_track, self.south_section_id, 0) first_north = create_event(self.first_track, self.north_section_id, 1) @@ -393,12 +400,13 @@ def __create_single_track_multiple_selection( self.south_to_east, EventPair(first_south, first_east), ) - ] + ], + mock_factory, ) return (events, self.flows, expected_result) def __create_complex_test_case( - self, + self, mock_factory: Mock ) -> tuple[list[Event], list[Flow], RoadUserAssignments]: first_south = create_event(self.first_track, self.south_section_id, 0) second_south = create_event(self.second_track, self.south_section_id, 1) @@ -466,12 +474,13 @@ def __create_complex_test_case( self.north_to_south, EventPair(sixth_north, sixth_south), ), - ] + ], + mock_factory, ) return (events, self.flows, expected_result) def __create_same_frame_events( - self, + self, mock_factory: Mock ) -> tuple[list[Event], list[Flow], RoadUserAssignments]: first_south = create_event( track_id=self.first_track, @@ -497,7 +506,8 @@ def __create_same_frame_events( self.south_to_north, EventPair(first_south, first_north), ) - ] + ], + mock_factory, ) return events, self.flows, expected_result @@ -816,10 +826,10 @@ def __create_multiple_candidate_assignment( return self.first_track.id, track_classification, candidates, expected_result -def create_assignment_test_cases() -> ( - list[tuple[list[Event], list[Flow], RoadUserAssignments]] -): - return TestCaseBuilder().build_assignment_test_cases() +def create_assignment_test_cases( + mock_factory: Mock, +) -> list[tuple[list[Event], list[Flow], RoadUserAssignments]]: + return TestCaseBuilder().build_assignment_test_cases(mock_factory) class TestFilterBySectionEnterEvent: @@ -843,7 +853,7 @@ def test_assign_filters_by_section_enter_event(self) -> None: class TestSimpleRoadUserAssigner: @pytest.mark.parametrize( - "events, flows, expected_result", create_assignment_test_cases() + "events, flows, expected_result", create_assignment_test_cases(mock_factory()) ) def test_run( self, @@ -854,7 +864,7 @@ def test_run( """ https://openproject.platomo.de/projects/otanalytics/work_packages/6321/activity """ - analysis = SimpleRoadUserAssigner() + analysis = SimpleRoadUserAssigner(mock_factory()) result = analysis.assign(events, flows) assert result == expected_result @@ -884,10 +894,12 @@ def _create_flow_selection(self) -> Mock: return flow_selection def _create_target(self, flow_selection: Mock) -> RoadUserAssigner: - return SimpleRoadUserAssigner(flow_selection=flow_selection) + return SimpleRoadUserAssigner( + flow_selection=flow_selection, track_id_set_factory=mock_factory() + ) def test_default_flow_selection(self) -> None: - target = SimpleRoadUserAssigner() + target = SimpleRoadUserAssigner(mock_factory()) assert isinstance(target._flow_selection, MaxDurationFlowSelection) @@ -904,7 +916,9 @@ def test_tag(self) -> None: bike_assignment = Mock(spec=RoadUserAssignment) mode = Mock(spec=Tagger) mode.create_tag.side_effect = [first_groupd, second_groupd] - assignments = RoadUserAssignments([car_assignment, bike_assignment]) + assignments = RoadUserAssignments( + [car_assignment, bike_assignment], mock_factory() + ) tagged = assignments.tag(by=mode) @@ -916,6 +930,25 @@ def test_tag(self) -> None: ) +class TestRoadUserAssignments: + def test_ids(self) -> None: + car_id = "car-id" + bike_id = "bike-id" + car_assignment = Mock(spec=RoadUserAssignment) + car_assignment.road_user = car_id + bike_assignment = Mock(spec=RoadUserAssignment) + bike_assignment.road_user = bike_id + track_id_set = Mock(spec=TrackIdSet) + factory = mock_factory() + factory.create.return_value = track_id_set + assignments = RoadUserAssignments([car_assignment, bike_assignment], factory) + + ids = assignments.road_user_ids + + assert ids == track_id_set + factory.create.assert_called_once_with({car_id, bike_id}) + + class TestModeTagger: def test_create_tag_existing_track(self, track: Track) -> None: flow = Mock(spec=Flow) diff --git a/tests/OTAnalytics/plugin_parser/test_road_user_assignment_export.py b/tests/OTAnalytics/plugin_parser/test_road_user_assignment_export.py index f68ca16f0..a738ce485 100644 --- a/tests/OTAnalytics/plugin_parser/test_road_user_assignment_export.py +++ b/tests/OTAnalytics/plugin_parser/test_road_user_assignment_export.py @@ -30,6 +30,7 @@ def test_export( ) -> None: save_path = test_data_tmp_dir / "road_user_assignments.csv" + mock_factory = Mock() section_repository = Mock() get_all_tracks = Mock() builder = RoadUserAssignmentBuilder() @@ -52,7 +53,7 @@ def test_export( ) exporter.export( RoadUserAssignments( - [first_road_user_assignment, second_road_user_assignment] + [first_road_user_assignment, second_road_user_assignment], mock_factory ), OVERWRITE, ) diff --git a/tests/OTAnalytics/plugin_ui/test_cli.py b/tests/OTAnalytics/plugin_ui/test_cli.py index 474a5faa3..285a88e78 100644 --- a/tests/OTAnalytics/plugin_ui/test_cli.py +++ b/tests/OTAnalytics/plugin_ui/test_cli.py @@ -435,7 +435,10 @@ def cli_dependencies_old(self) -> dict[str, Any]: create_events = CreateEvents( clear_all_events, create_intersection_events, create_scene_events ) - assigner = FilterBySectionEnterEvent(SimpleRoadUserAssigner()) + track_id_set_factory = PolarsTrackIdSetFactory() + assigner = FilterBySectionEnterEvent( + SimpleRoadUserAssigner(track_id_set_factory) + ) traffic_counting = TrafficCounting( event_repository, flow_repository, @@ -533,7 +536,10 @@ def cli_dependencies(self) -> dict[str, Any]: create_events = CreateEvents( clear_all_events, create_intersection_events, create_scene_events ) - assigner = FilterBySectionEnterEvent(SimpleRoadUserAssigner()) + track_id_set_factory = PolarsTrackIdSetFactory() + assigner = FilterBySectionEnterEvent( + SimpleRoadUserAssigner(track_id_set_factory) + ) traffic_counting = TrafficCounting( event_repository, flow_repository, @@ -550,9 +556,8 @@ def cli_dependencies(self) -> dict[str, Any]: ) get_all_tracks = GetAllTracks(track_repository) get_cutting_sections = GetCuttingSections(section_repository) - track_id_set_factory = PolarsTrackIdSetFactory() tracks_assigned_to_all_flows = TracksAssignedToAllFlows( - SimpleRoadUserAssigner(), + SimpleRoadUserAssigner(track_id_set_factory), event_repository, flow_repository, track_id_set_factory, From 002c8eff5ca48a3fb8c48bc65cc976ad034ee84c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 15:46:34 +0200 Subject: [PATCH 079/128] Fix clean architecture violation by using TrackIdSet instead of iterable --- OTAnalytics/application/use_cases/track_repository.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index 6c21c1b91..b1ec5fb8f 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -10,7 +10,6 @@ TrackFileRepository, TrackRepository, ) -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet class AllTrackIdsProvider(TrackIdProvider): @@ -142,16 +141,16 @@ class RemoveTracks: def __init__(self, track_repository: TrackRepository) -> None: self._track_repository = track_repository - def __call__(self, track_ids: Iterable[TrackId]) -> None: + def __call__(self, track_ids: TrackIdSet) -> None: """Remove tracks from track repository. Tracks that do not exist in the repository will be skipped. Args: - track_ids (Iterable[TrackId]): ids of tracks to be removed. + track_ids (TrackIdSet): ids of tracks to be removed. """ try: - self._track_repository.remove_multiple(PythonTrackIdSet(track_ids)) + self._track_repository.remove_multiple(track_ids) except RemoveMultipleTracksError as cause: logger().info(cause) From 361ba4cdeb73269b2df607590b98a9fd4a3e0cf8 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 15:58:54 +0200 Subject: [PATCH 080/128] Fix clean architecture violation by using TrackIdSet instead of iterable --- .../application/use_cases/track_repository.py | 4 +- OTAnalytics/domain/track_repository.py | 30 +++----------- tests/OTAnalytics/application/test_state.py | 15 +++++-- .../use_cases/test_inside_cutting_section.py | 4 +- .../domain/test_track_repository.py | 39 +++++++------------ .../track_visualization/test_track_viz.py | 25 +++++++++--- 6 files changed, 52 insertions(+), 65 deletions(-) diff --git a/OTAnalytics/application/use_cases/track_repository.py b/OTAnalytics/application/use_cases/track_repository.py index b1ec5fb8f..ec53962f4 100644 --- a/OTAnalytics/application/use_cases/track_repository.py +++ b/OTAnalytics/application/use_cases/track_repository.py @@ -84,11 +84,11 @@ class GetAllTrackIds: def __init__(self, track_repository: TrackRepository) -> None: self._track_repository = track_repository - def __call__(self) -> Iterable[TrackId]: + def __call__(self) -> TrackIdSet: """Get all track ids from the track repository. Returns: - Iterable[TrackId]: the track ids. + TrackIdSet: the track ids. """ return self._track_repository.get_all_ids() diff --git a/OTAnalytics/domain/track_repository.py b/OTAnalytics/domain/track_repository.py index a64686e1a..6f57b3beb 100644 --- a/OTAnalytics/domain/track_repository.py +++ b/OTAnalytics/domain/track_repository.py @@ -12,7 +12,6 @@ TrackDataset, TrackIdSet, ) -from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet @dataclass(frozen=True) @@ -21,12 +20,12 @@ class TrackRepositoryEvent: removed: TrackIdSet @staticmethod - def create_added(tracks: Iterable[TrackId]) -> "TrackRepositoryEvent": - return TrackRepositoryEvent(PythonTrackIdSet(tracks), EmptyTrackIdSet()) + def create_added(tracks: TrackIdSet) -> "TrackRepositoryEvent": + return TrackRepositoryEvent(tracks, EmptyTrackIdSet()) @staticmethod - def create_removed(tracks: Iterable[TrackId]) -> "TrackRepositoryEvent": - return TrackRepositoryEvent(EmptyTrackIdSet(), PythonTrackIdSet(tracks)) + def create_removed(tracks: TrackIdSet) -> "TrackRepositoryEvent": + return TrackRepositoryEvent(EmptyTrackIdSet(), tracks) class TrackListObserver(ABC): @@ -179,25 +178,6 @@ def get_all_ids(self) -> TrackIdSet: """ return self._dataset.track_ids - def remove(self, track_id: TrackId) -> None: - """Remove track by its id and notify observers - - Raises: - TrackRemoveError: if track does not exist in repository. - - Args: - track_id (TrackId): the id of the track to be removed. - """ - try: - self._dataset = self._dataset.remove(track_id) - except KeyError: - raise TrackRemoveError( - track_id, f"Trying to remove non existing track with id '{track_id.id}'" - ) - # TODO: Pass removed track id to notify when moving observers to - # application layer - self.observers.notify(TrackRepositoryEvent.create_removed([track_id])) - def remove_multiple(self, track_ids: TrackIdSet) -> None: not_existing_tracks = track_ids.difference(self._dataset.track_ids) if not_existing_tracks: @@ -219,7 +199,7 @@ def clear(self) -> None: """ Clear the repository and inform the observers about the empty repository. """ - removed = list(self._dataset.track_ids) + removed = self._dataset.track_ids self._dataset = self._dataset.clear() self.observers.notify(TrackRepositoryEvent.create_removed(removed)) diff --git a/tests/OTAnalytics/application/test_state.py b/tests/OTAnalytics/application/test_state.py index 9b03f950f..f56911176 100644 --- a/tests/OTAnalytics/application/test_state.py +++ b/tests/OTAnalytics/application/test_state.py @@ -40,6 +40,7 @@ TrackRepositoryEvent, ) from OTAnalytics.domain.video import VideoMetadata +from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet FIRST_START_DATE = datetime( year=2019, @@ -82,7 +83,9 @@ def test_update_selected_track_on_notify_tracks(self) -> None: state = TrackState() state.notify_tracks( - TrackRepositoryEvent.create_added([first_track, second_track]) + TrackRepositoryEvent.create_added( + PythonTrackIdSet([first_track, second_track]) + ) ) assert state.selected_track in [first_track, second_track] @@ -91,7 +94,9 @@ def test_update_selected_track_on_notify_tracks_with_empty_list(self) -> None: first_track = TrackId("1") state = TrackState() - state.notify_tracks(TrackRepositoryEvent.create_added([first_track])) + state.notify_tracks( + TrackRepositoryEvent.create_added(PythonTrackIdSet([first_track])) + ) state.notify_tracks(TrackRepositoryEvent(EmptyTrackIdSet(), EmptyTrackIdSet())) assert state.selected_track is None @@ -99,7 +104,9 @@ def test_update_selected_track_on_notify_tracks_with_empty_list(self) -> None: def test_reset(self) -> None: first_track = TrackId("1") target = TrackState() - target.notify_tracks(TrackRepositoryEvent.create_added([first_track])) + target.notify_tracks( + TrackRepositoryEvent.create_added(PythonTrackIdSet([first_track])) + ) target.reset() @@ -237,7 +244,7 @@ def test_update_image(self) -> None: updater = TrackImageUpdater( datastore, track_view_state, section_state, flow_state, plotter ) - tracks: list[TrackId] = [track_id] + tracks = PythonTrackIdSet([track_id]) updater.notify_tracks(TrackRepositoryEvent.create_added(tracks)) diff --git a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py index 909429b34..cc3151600 100644 --- a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py +++ b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py @@ -31,12 +31,12 @@ def track_ids() -> TrackIdSet: @pytest.fixture def add_track_event() -> TrackRepositoryEvent: - return TrackRepositoryEvent.create_added([TrackId("1")]) + return TrackRepositoryEvent.create_added(PythonTrackIdSet([TrackId("1")])) @pytest.fixture def remove_track_event() -> TrackRepositoryEvent: - return TrackRepositoryEvent.create_removed([TrackId("2")]) + return TrackRepositoryEvent.create_removed(PythonTrackIdSet([TrackId("2")])) @pytest.fixture diff --git a/tests/OTAnalytics/domain/test_track_repository.py b/tests/OTAnalytics/domain/test_track_repository.py index cd35afaa7..2b1fa88be 100644 --- a/tests/OTAnalytics/domain/test_track_repository.py +++ b/tests/OTAnalytics/domain/test_track_repository.py @@ -4,7 +4,7 @@ import pytest from OTAnalytics.domain.track import Track, TrackId -from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset +from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet, TrackDataset from OTAnalytics.domain.track_repository import ( TrackFileRepository, TrackListObserver, @@ -43,7 +43,7 @@ def track_2(self) -> Mock: def test_add_all(self, track_1: Mock, track_2: Mock) -> None: track_ids_property = PropertyMock( - return_value=frozenset([track_1.id, track_2.id]) + return_value=PythonTrackIdSet([track_1.id, track_2.id]) ) tracks = MagicMock(spec=TrackDataset) tracks.__len__.return_value = 2 @@ -63,7 +63,9 @@ def test_add_all(self, track_1: Mock, track_2: Mock) -> None: dataset.add_all.assert_called_with(tracks) observer.notify_tracks.assert_called_with( - TrackRepositoryEvent.create_added([track_1.id, track_2.id]) + TrackRepositoryEvent.create_added( + PythonTrackIdSet([track_1.id, track_2.id]) + ) ) def test_add_nothing(self) -> None: @@ -93,7 +95,7 @@ def test_clear(self, track_1: Track, track_2: Track) -> None: cleared_dataset = Mock(spec=TrackDataset) dataset = Mock(spec=TrackDataset) type(dataset).track_ids = PropertyMock( - return_value=frozenset([track_1.id, track_2.id]) + return_value=PythonTrackIdSet([track_1.id, track_2.id]) ) dataset.clear.return_value = cleared_dataset observer = Mock(spec=TrackListObserver) @@ -104,11 +106,15 @@ def test_clear(self, track_1: Track, track_2: Track) -> None: assert repository._dataset == cleared_dataset assert observer.notify_tracks.call_args_list == [ - call(TrackRepositoryEvent.create_removed([track_1.id, track_2.id])) + call( + TrackRepositoryEvent.create_removed( + PythonTrackIdSet([track_1.id, track_2.id]) + ) + ) ] def test_get_all_ids(self, track_1: Mock, track_2: Mock) -> None: - ids: frozenset[TrackId] = frozenset() + ids = EmptyTrackIdSet() dataset = Mock(spec=TrackDataset) type(dataset).track_ids = PropertyMock(return_value=ids) repository = TrackRepository(dataset) @@ -117,25 +123,6 @@ def test_get_all_ids(self, track_1: Mock, track_2: Mock) -> None: assert actual_ids is ids - def test_remove(self, track_1: Track, track_2: Track) -> None: - dataset = Mock(spec=TrackDataset) - dataset.remove.return_value = dataset - repository = TrackRepository(dataset) - - observer = Mock(spec=TrackListObserver) - repository.register_tracks_observer(observer) - - repository.remove(track_1.id) - assert len(dataset.remove.call_args_list) == 1 - assert call(track_1.id) in dataset.remove.call_args_list - repository.remove(track_2.id) - assert call(track_2.id) in dataset.remove.call_args_list - - assert observer.notify_tracks.call_args_list == [ - call(TrackRepositoryEvent.create_removed([track_1.id])), - call(TrackRepositoryEvent.create_removed([track_2.id])), - ] - def test_remove_multiple(self, track_1: Track, track_2: Track) -> None: tracks_to_remove = PythonTrackIdSet([track_1.id, track_2.id]) updated_dataset = Mock(spec=TrackDataset) @@ -216,7 +203,7 @@ def test_remove_by_original_ids(self) -> None: given_observer = Mock() given_dataset = Mock() updated_dataset = Mock() - removed_ids = frozenset([TrackId("actual-1")]) + removed_ids = PythonTrackIdSet([TrackId("actual-1")]) given_dataset.remove_by_original_ids.return_value = ( updated_dataset, diff --git a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py index 43e9aa0dd..f843675de 100644 --- a/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py +++ b/tests/OTAnalytics/plugin_prototypes/track_visualization/test_track_viz.py @@ -27,6 +27,7 @@ PythonDetection, PythonTrack, PythonTrackDataset, + PythonTrackIdSet, ) from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, @@ -252,21 +253,27 @@ def test_notify_tracks_clear_cache(self, track_1: Track) -> None: """Test clearing cache.""" provider = self.set_up_provider([track_1], []) - provider.notify_tracks(TrackRepositoryEvent.create_removed([track_1.id])) + provider.notify_tracks( + TrackRepositoryEvent.create_removed(PythonTrackIdSet([track_1.id])) + ) self.check_expected_ids(provider, []) def test_notify_update_add(self, track_1: Track, track_2: Track) -> None: """Test adding track to non-empty cache.""" provider = self.set_up_provider([track_1], [track_2]) - provider.notify_tracks(TrackRepositoryEvent.create_added([track_2.id])) + provider.notify_tracks( + TrackRepositoryEvent.create_added(PythonTrackIdSet([track_2.id])) + ) self.check_expected_ids(provider, [track_1, track_2]) def test_notify_update_add_first(self, track_2: Track) -> None: """Test adding first track to cache.""" provider = self.set_up_provider([], [track_2]) - provider.notify_tracks(TrackRepositoryEvent.create_added([track_2.id])) + provider.notify_tracks( + TrackRepositoryEvent.create_added(PythonTrackIdSet([track_2.id])) + ) self.check_expected_ids(provider, [track_2]) def test_notify_update_add_multiple_first( @@ -276,21 +283,27 @@ def test_notify_update_add_multiple_first( provider = self.set_up_provider([], [track_2, track_1]) provider.notify_tracks( - TrackRepositoryEvent.create_added([track_2.id, track_1.id]) + TrackRepositoryEvent.create_added( + PythonTrackIdSet([track_2.id, track_1.id]) + ) ) self.check_expected_ids(provider, [track_2, track_1]) def test_notify_update_existing(self, track_1: Track, track_2: Track) -> None: provider = self.set_up_provider([track_1, track_2], [track_1]) - provider.notify_tracks(TrackRepositoryEvent.create_added([track_1.id])) + provider.notify_tracks( + TrackRepositoryEvent.create_added(PythonTrackIdSet([track_1.id])) + ) self.check_expected_ids(provider, [track_1, track_2]) def test_notify_update_mixed(self, track_1: Track, track_2: Track) -> None: provider = self.set_up_provider([track_2], [track_1, track_2]) provider.notify_tracks( - TrackRepositoryEvent.create_added([track_1.id, track_2.id]) + TrackRepositoryEvent.create_added( + PythonTrackIdSet([track_1.id, track_2.id]) + ) ) self.check_expected_ids(provider, [track_1, track_2]) From 48973e1b94954998ef90c2e757950359055a79ab Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 16:12:07 +0200 Subject: [PATCH 081/128] Extract classes into separate file --- .../filter_pandas_track_store.py | 324 ++++++++++++++++++ .../get_current_frame_track_segments.py | 8 +- OTAnalytics/plugin_datastore/track_store.py | 301 ---------------- .../plugin_datastore/test_track_store.py | 6 +- tests/benchmark_otanalytics.py | 4 +- .../utils/builders/track_dataset_provider.py | 8 +- 6 files changed, 339 insertions(+), 312 deletions(-) create mode 100644 OTAnalytics/plugin_datastore/filter_pandas_track_store.py diff --git a/OTAnalytics/plugin_datastore/filter_pandas_track_store.py b/OTAnalytics/plugin_datastore/filter_pandas_track_store.py new file mode 100644 index 000000000..af8d17763 --- /dev/null +++ b/OTAnalytics/plugin_datastore/filter_pandas_track_store.py @@ -0,0 +1,324 @@ +from abc import abstractmethod +from typing import Iterable, Sequence + +from pandas import DataFrame + +from OTAnalytics.application.logger import logger +from OTAnalytics.domain import track +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.section import Section +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset.filtered_track_dataset import ( + FilterByClassTrackDataset, + FilteredTrackDataset, +) +from OTAnalytics.domain.track_dataset.track_dataset import ( + TRACK_GEOMETRY_FACTORY, + TrackIdSet, +) +from OTAnalytics.plugin_datastore.track_store import ( + LEVEL_TRACK_ID, + MINIMUM_DETECTIONS, + RANK, + PandasDataFrameProvider, + PandasTrackClassificationCalculator, + PandasTrackDataset, + get_rows_by_track_ids, +) + + +class FilteredPandasTrackDataset( + FilteredTrackDataset, PandasTrackDataset, PandasDataFrameProvider +): + + @property + def track_geometry_factory(self) -> TRACK_GEOMETRY_FACTORY: + return self._other.track_geometry_factory + + @property + def calculator(self) -> PandasTrackClassificationCalculator: + return self._other.calculator + + def __init__(self, other: PandasTrackDataset) -> None: + self._other = other + + def add_all(self, other: Iterable[Track]) -> PandasTrackDataset: + return self.wrap(self._other.add_all(other)) + + def remove(self, track_id: TrackId) -> PandasTrackDataset: + return self.wrap(self._other.remove(track_id)) + + def remove_multiple(self, track_ids: TrackIdSet) -> PandasTrackDataset: + return self.wrap(self._other.remove_multiple(track_ids)) + + def clear(self) -> PandasTrackDataset: + return self.wrap(self._other.clear()) + + def split(self, chunks: int) -> Sequence[PandasTrackDataset]: + return [self.wrap(dataset) for dataset in self._other.split(chunks)] + + def calculate_geometries_for( + self, offsets: Iterable[RelativeOffsetCoordinate] + ) -> None: + self._other.calculate_geometries_for(offsets) + + def cut_with_section( + self, section: Section, offset: RelativeOffsetCoordinate + ) -> tuple[PandasTrackDataset, TrackIdSet]: + dataset, original_track_ids = self._other.cut_with_section(section, offset) + return self.wrap(dataset), original_track_ids + + @abstractmethod + def _filter(self) -> PandasTrackDataset: + raise NotImplementedError + + @abstractmethod + def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: + raise NotImplementedError + + def get_data(self) -> DataFrame: + return self._filter().get_data() + + def revert_cuts_for( + self, original_track_ids: TrackIdSet + ) -> tuple[PandasTrackDataset, TrackIdSet, TrackIdSet]: + reverted_dataset, reverted_ids, cut_ids = self._other.revert_cuts_for( + original_track_ids + ) + return self.wrap(reverted_dataset), reverted_ids, cut_ids + + def remove_by_original_ids( + self, original_ids: TrackIdSet + ) -> tuple["PandasTrackDataset", TrackIdSet]: + updated_dataset, removed_ids = self._other.remove_by_original_ids(original_ids) + return self.wrap(updated_dataset), removed_ids + + +class FilterByClassPandasTrackDataset( + FilteredPandasTrackDataset, FilterByClassTrackDataset +): + @property + def include_classes(self) -> frozenset[str]: + return self._include_classes + + @property + def exclude_classes(self) -> frozenset[str]: + return self._exclude_classes + + def __init__( + self, + other: PandasTrackDataset, + include_classes: frozenset[str], + exclude_classes: frozenset[str], + ) -> None: + super().__init__(other) + self._include_classes = include_classes + self._exclude_classes = exclude_classes + self._cache: PandasTrackDataset | None = None + + def _filter(self) -> PandasTrackDataset: + """Filter TrackDataset by classifications. + + IMPORTANT: Classifications contained in the include_classes will not be + removed even if they appear in the set of exclude_classes. + Furthermore, the whitelist will not be applied if empty. + + Returns: + TrackDataset: the filtered dataset. + """ + if not self.include_classes and not self._exclude_classes: + return self._other + + if self._cache is not None: + return self._cache + + if self.include_classes: + logger().info( + "Apply 'include-classes' filter to filter tracks: " + f"{self.include_classes}" + "\n'exclude-classes' filter is not used" + ) + filtered_dataset = self._get_dataset_with_classes( + list(self._other.classifications & self.include_classes) + ) + elif self.exclude_classes: + logger().info( + "Apply 'exclude-classes' filter to filter tracks: " + f"{self.exclude_classes}" + ) + filtered_dataset = self._get_dataset_with_classes( + list(self._other.classifications - self.exclude_classes) + ) + else: + return self._other + self._cache = filtered_dataset + return filtered_dataset + + def _get_dataset_with_classes(self, classes: list[str]) -> PandasTrackDataset: + if self._other.empty: + return self._other + dataset = self._other.get_data() + mask = dataset[track.TRACK_CLASSIFICATION].isin(classes) + filtered_df = dataset[mask] + # The pandas Index does not implement the Sequence interface, which causes + # compatibility issues with the PandasTrackDataset._remove_from_geometry method + # when trying to remove geometries for tracks that have been deleted. + # To address this, we invalidate the entire geometry cache rather than + # attempting selective removal. + # This approach is acceptable because track removal only occurs when + # cutting tracks, which is a rare use case. + + return PandasTrackDataset( + track_geometry_factory=self._other.track_geometry_factory, + dataset=filtered_df, + geometry_datasets=None, + calculator=self._other.calculator, + ) + + def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: + return FilterByClassPandasTrackDataset( + other, self.include_classes, self.exclude_classes + ) + + +class FilterByIdPandasTrackDataset(FilteredPandasTrackDataset): + """ + Represents a dataset that filters tracks by specific track IDs. + + This class provides functionality to filter a Pandas-based track dataset by a + list of track IDs. It utilizes caching to optimize repeated filtering operations, + manages dataset consistency, and ensures efficient geometry cache invalidation + for cases such as track removal. + """ + + def __init__(self, other: PandasTrackDataset, track_ids: list[str]) -> None: + super().__init__(other) + self._included_track_ids = track_ids + self._cache: PandasTrackDataset | None = None + + def _filter(self) -> PandasTrackDataset: + """ + Filters the track dataset based on included track IDs. + + This method applies a filter on a dataset to include only the tracks + specified in `_included_track_ids`. If there are no specified track IDs, + it returns the original dataset (`_other`). It employs caching to store the + filtered dataset for future use. + + Returns: + TrackDataset: The filtered dataset containing only the specified track IDs. + If no track IDs are provided, the original dataset is returned. + """ + if not self._included_track_ids: + return self._other + + if self._cache is not None: + return self._cache + + if self._included_track_ids: + logger().info( + "Apply 'track-ids' filter to filter tracks: " + f"{self._included_track_ids}" + ) + filtered_dataset = self._get_dataset_with( + track_ids=self._included_track_ids + ) + else: + return self._other + self._cache = filtered_dataset + return filtered_dataset + + def _get_dataset_with(self, track_ids: list[str]) -> PandasTrackDataset: + if self._other.empty: + return self._other + dataset = self._other.get_data() + filtered_df = get_rows_by_track_ids(dataset, track_ids) + # The pandas Index does not implement the Sequence interface, which causes + # compatibility issues with the PandasTrackDataset._remove_from_geometry method + # when trying to remove geometries for tracks that have been deleted. + # To address this, we invalidate the entire geometry cache rather than + # attempting selective removal. + # This approach is acceptable because track removal only occurs when + # cutting tracks, which is a rare use case. + + return PandasTrackDataset( + track_geometry_factory=self._other.track_geometry_factory, + dataset=filtered_df, + geometry_datasets=None, + calculator=self._other.calculator, + ) + + def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: + return FilterByIdPandasTrackDataset(other, self._included_track_ids) + + +class FilterLastNDetectionsPandasTrackDataset(FilteredPandasTrackDataset): + def __init__(self, other: PandasTrackDataset, n: int) -> None: + super().__init__(other) + self._n = n + self._cache: PandasTrackDataset | None = None + + def _filter(self) -> PandasTrackDataset: + """ + Filters the track dataset based on included track IDs. + + This method applies a filter on a dataset to include only the tracks + specified in `_included_track_ids`. If there are no specified track IDs, + it returns the original dataset (`_other`). It employs caching to store the + filtered dataset for future use. + + Returns: + TrackDataset: The filtered dataset containing only the specified track IDs. + If no track IDs are provided, the original dataset is returned. + """ + if self._cache is not None: + return self._cache + + logger().info( + f"Limit number of detections per track to last {self._n} of them." + ) + filtered_dataset = self._get_filtered_dataset() + self._cache = filtered_dataset + return filtered_dataset + + def _get_filtered_dataset(self) -> PandasTrackDataset: + if self._other.empty: + return self._other + dataset = self._other.get_data() + filtered_df = get_exactly_two_latest_occurrences_per_id(dataset, self._n) + # The pandas Index does not implement the Sequence interface, which causes + # compatibility issues with the PandasTrackDataset._remove_from_geometry method + # when trying to remove geometries for tracks that have been deleted. + # To address this, we invalidate the entire geometry cache rather than + # attempting selective removal. + # This approach is acceptable because track removal only occurs when + # cutting tracks, which is a rare use case. + + return PandasTrackDataset( + track_geometry_factory=self._other.track_geometry_factory, + dataset=filtered_df, + geometry_datasets=None, + calculator=self._other.calculator, + ) + + def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: + return FilterLastNDetectionsPandasTrackDataset(other, self._n) + + +def get_latest_occurrences(dataframe: DataFrame, last_n: int) -> DataFrame: + index_names = dataframe.index.names + result = dataframe.reset_index() + result.loc[:, RANK] = result.groupby(track.TRACK_ID)[track.OCCURRENCE].rank( + method="first", ascending=False + ) + result = result[result[RANK] <= last_n].drop(RANK, axis=1).reset_index(drop=True) + return result.set_index(index_names) + + +def get_exactly_two_latest_occurrences_per_id( + dataframe: DataFrame, last_n: int +) -> DataFrame: + counts = dataframe.index.get_level_values(LEVEL_TRACK_ID).value_counts() + valid_ids = counts[counts >= MINIMUM_DETECTIONS].index + filtered_df = get_rows_by_track_ids(dataframe=dataframe, track_ids=valid_ids) + return get_latest_occurrences(filtered_df, last_n=last_n) diff --git a/OTAnalytics/plugin_datastore/get_current_frame_track_segments.py b/OTAnalytics/plugin_datastore/get_current_frame_track_segments.py index 8a642bd5d..fe4406813 100644 --- a/OTAnalytics/plugin_datastore/get_current_frame_track_segments.py +++ b/OTAnalytics/plugin_datastore/get_current_frame_track_segments.py @@ -1,12 +1,12 @@ from OTAnalytics.application.use_cases.create_intersection_events import GetTracks from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset -from OTAnalytics.plugin_datastore.pandas_track_dataset_factory import ( - PandasTrackDatasetFactory, -) -from OTAnalytics.plugin_datastore.track_store import ( +from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( FilterByIdPandasTrackDataset, FilterLastNDetectionsPandasTrackDataset, ) +from OTAnalytics.plugin_datastore.pandas_track_dataset_factory import ( + PandasTrackDatasetFactory, +) NUMBER_OF_DETECTIONS_TO_FORM_SEGMENT = 2 diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index eee57c274..189a04278 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -16,10 +16,6 @@ from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Detection, Track, TrackId, unpack -from OTAnalytics.domain.track_dataset.filtered_track_dataset import ( - FilterByClassTrackDataset, - FilteredTrackDataset, -) from OTAnalytics.domain.track_dataset.track_dataset import ( END_FRAME, END_OCCURRENCE, @@ -771,303 +767,6 @@ def to_domain_ids(raw_ids: Iterable[str]) -> TrackIdSet: return PythonTrackIdSet((TrackId(raw_id) for raw_id in raw_ids)) -class FilteredPandasTrackDataset( - FilteredTrackDataset, PandasTrackDataset, PandasDataFrameProvider -): - - @property - def track_geometry_factory(self) -> TRACK_GEOMETRY_FACTORY: - return self._other.track_geometry_factory - - @property - def calculator(self) -> PandasTrackClassificationCalculator: - return self._other.calculator - - def __init__(self, other: PandasTrackDataset) -> None: - self._other = other - - def add_all(self, other: Iterable[Track]) -> PandasTrackDataset: - return self.wrap(self._other.add_all(other)) - - def remove(self, track_id: TrackId) -> PandasTrackDataset: - return self.wrap(self._other.remove(track_id)) - - def remove_multiple(self, track_ids: TrackIdSet) -> PandasTrackDataset: - return self.wrap(self._other.remove_multiple(track_ids)) - - def clear(self) -> PandasTrackDataset: - return self.wrap(self._other.clear()) - - def split(self, chunks: int) -> Sequence[PandasTrackDataset]: - return [self.wrap(dataset) for dataset in self._other.split(chunks)] - - def calculate_geometries_for( - self, offsets: Iterable[RelativeOffsetCoordinate] - ) -> None: - self._other.calculate_geometries_for(offsets) - - def cut_with_section( - self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple[PandasTrackDataset, TrackIdSet]: - dataset, original_track_ids = self._other.cut_with_section(section, offset) - return self.wrap(dataset), original_track_ids - - @abstractmethod - def _filter(self) -> PandasTrackDataset: - raise NotImplementedError - - @abstractmethod - def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: - raise NotImplementedError - - def get_data(self) -> DataFrame: - return self._filter().get_data() - - def revert_cuts_for( - self, original_track_ids: TrackIdSet - ) -> tuple[PandasTrackDataset, TrackIdSet, TrackIdSet]: - reverted_dataset, reverted_ids, cut_ids = self._other.revert_cuts_for( - original_track_ids - ) - return self.wrap(reverted_dataset), reverted_ids, cut_ids - - def remove_by_original_ids( - self, original_ids: TrackIdSet - ) -> tuple["PandasTrackDataset", TrackIdSet]: - updated_dataset, removed_ids = self._other.remove_by_original_ids(original_ids) - return self.wrap(updated_dataset), removed_ids - - -class FilterByClassPandasTrackDataset( - FilteredPandasTrackDataset, FilterByClassTrackDataset -): - @property - def include_classes(self) -> frozenset[str]: - return self._include_classes - - @property - def exclude_classes(self) -> frozenset[str]: - return self._exclude_classes - - def __init__( - self, - other: PandasTrackDataset, - include_classes: frozenset[str], - exclude_classes: frozenset[str], - ) -> None: - super().__init__(other) - self._include_classes = include_classes - self._exclude_classes = exclude_classes - self._cache: PandasTrackDataset | None = None - - def _filter(self) -> PandasTrackDataset: - """Filter TrackDataset by classifications. - - IMPORTANT: Classifications contained in the include_classes will not be - removed even if they appear in the set of exclude_classes. - Furthermore, the whitelist will not be applied if empty. - - Returns: - TrackDataset: the filtered dataset. - """ - if not self.include_classes and not self._exclude_classes: - return self._other - - if self._cache is not None: - return self._cache - - if self.include_classes: - logger().info( - "Apply 'include-classes' filter to filter tracks: " - f"{self.include_classes}" - "\n'exclude-classes' filter is not used" - ) - filtered_dataset = self._get_dataset_with_classes( - list(self._other.classifications & self.include_classes) - ) - elif self.exclude_classes: - logger().info( - "Apply 'exclude-classes' filter to filter tracks: " - f"{self.exclude_classes}" - ) - filtered_dataset = self._get_dataset_with_classes( - list(self._other.classifications - self.exclude_classes) - ) - else: - return self._other - self._cache = filtered_dataset - return filtered_dataset - - def _get_dataset_with_classes(self, classes: list[str]) -> PandasTrackDataset: - if self._other.empty: - return self._other - dataset = self._other.get_data() - mask = dataset[track.TRACK_CLASSIFICATION].isin(classes) - filtered_df = dataset[mask] - # The pandas Index does not implement the Sequence interface, which causes - # compatibility issues with the PandasTrackDataset._remove_from_geometry method - # when trying to remove geometries for tracks that have been deleted. - # To address this, we invalidate the entire geometry cache rather than - # attempting selective removal. - # This approach is acceptable because track removal only occurs when - # cutting tracks, which is a rare use case. - - return PandasTrackDataset( - track_geometry_factory=self._other.track_geometry_factory, - dataset=filtered_df, - geometry_datasets=None, - calculator=self._other.calculator, - ) - - def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: - return FilterByClassPandasTrackDataset( - other, self.include_classes, self.exclude_classes - ) - - -class FilterByIdPandasTrackDataset(FilteredPandasTrackDataset): - """ - Represents a dataset that filters tracks by specific track IDs. - - This class provides functionality to filter a Pandas-based track dataset by a - list of track IDs. It utilizes caching to optimize repeated filtering operations, - manages dataset consistency, and ensures efficient geometry cache invalidation - for cases such as track removal. - """ - - def __init__(self, other: PandasTrackDataset, track_ids: list[str]) -> None: - super().__init__(other) - self._included_track_ids = track_ids - self._cache: PandasTrackDataset | None = None - - def _filter(self) -> PandasTrackDataset: - """ - Filters the track dataset based on included track IDs. - - This method applies a filter on a dataset to include only the tracks - specified in `_included_track_ids`. If there are no specified track IDs, - it returns the original dataset (`_other`). It employs caching to store the - filtered dataset for future use. - - Returns: - TrackDataset: The filtered dataset containing only the specified track IDs. - If no track IDs are provided, the original dataset is returned. - """ - if not self._included_track_ids: - return self._other - - if self._cache is not None: - return self._cache - - if self._included_track_ids: - logger().info( - "Apply 'track-ids' filter to filter tracks: " - f"{self._included_track_ids}" - ) - filtered_dataset = self._get_dataset_with( - track_ids=self._included_track_ids - ) - else: - return self._other - self._cache = filtered_dataset - return filtered_dataset - - def _get_dataset_with(self, track_ids: list[str]) -> PandasTrackDataset: - if self._other.empty: - return self._other - dataset = self._other.get_data() - filtered_df = get_rows_by_track_ids(dataset, track_ids) - # The pandas Index does not implement the Sequence interface, which causes - # compatibility issues with the PandasTrackDataset._remove_from_geometry method - # when trying to remove geometries for tracks that have been deleted. - # To address this, we invalidate the entire geometry cache rather than - # attempting selective removal. - # This approach is acceptable because track removal only occurs when - # cutting tracks, which is a rare use case. - - return PandasTrackDataset( - track_geometry_factory=self._other.track_geometry_factory, - dataset=filtered_df, - geometry_datasets=None, - calculator=self._other.calculator, - ) - - def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: - return FilterByIdPandasTrackDataset(other, self._included_track_ids) - - -class FilterLastNDetectionsPandasTrackDataset(FilteredPandasTrackDataset): - def __init__(self, other: PandasTrackDataset, n: int) -> None: - super().__init__(other) - self._n = n - self._cache: PandasTrackDataset | None = None - - def _filter(self) -> PandasTrackDataset: - """ - Filters the track dataset based on included track IDs. - - This method applies a filter on a dataset to include only the tracks - specified in `_included_track_ids`. If there are no specified track IDs, - it returns the original dataset (`_other`). It employs caching to store the - filtered dataset for future use. - - Returns: - TrackDataset: The filtered dataset containing only the specified track IDs. - If no track IDs are provided, the original dataset is returned. - """ - if self._cache is not None: - return self._cache - - logger().info( - f"Limit number of detections per track to last {self._n} of them." - ) - filtered_dataset = self._get_filtered_dataset() - self._cache = filtered_dataset - return filtered_dataset - - def _get_filtered_dataset(self) -> PandasTrackDataset: - if self._other.empty: - return self._other - dataset = self._other.get_data() - filtered_df = get_exactly_two_latest_occurrences_per_id(dataset, self._n) - # The pandas Index does not implement the Sequence interface, which causes - # compatibility issues with the PandasTrackDataset._remove_from_geometry method - # when trying to remove geometries for tracks that have been deleted. - # To address this, we invalidate the entire geometry cache rather than - # attempting selective removal. - # This approach is acceptable because track removal only occurs when - # cutting tracks, which is a rare use case. - - return PandasTrackDataset( - track_geometry_factory=self._other.track_geometry_factory, - dataset=filtered_df, - geometry_datasets=None, - calculator=self._other.calculator, - ) - - def wrap(self, other: PandasTrackDataset) -> PandasTrackDataset: - return FilterLastNDetectionsPandasTrackDataset(other, self._n) - - -def get_latest_occurrences(dataframe: DataFrame, last_n: int) -> DataFrame: - index_names = dataframe.index.names - result = dataframe.reset_index() - result.loc[:, RANK] = result.groupby(track.TRACK_ID)[track.OCCURRENCE].rank( - method="first", ascending=False - ) - result = result[result[RANK] <= last_n].drop(RANK, axis=1).reset_index(drop=True) - return result.set_index(index_names) - - -def get_exactly_two_latest_occurrences_per_id( - dataframe: DataFrame, last_n: int -) -> DataFrame: - counts = dataframe.index.get_level_values(LEVEL_TRACK_ID).value_counts() - valid_ids = counts[counts >= MINIMUM_DETECTIONS].index - filtered_df = get_rows_by_track_ids(dataframe=dataframe, track_ids=valid_ids) - return get_latest_occurrences(filtered_df, last_n=last_n) - - def _assign_track_classification( data: DataFrame, calculator: PandasTrackClassificationCalculator ) -> DataFrame: diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 609ac99a5..38ff29dd9 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -17,6 +17,10 @@ TrackGeometryDataset, TrackIdSet, ) +from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( + FilterByIdPandasTrackDataset, + FilterLastNDetectionsPandasTrackDataset, +) from OTAnalytics.plugin_datastore.python_track_store import ( PythonTrack, PythonTrackDataset, @@ -28,8 +32,6 @@ from OTAnalytics.plugin_datastore.track_store import ( COLUMNS, INDEX_NAMES, - FilterByIdPandasTrackDataset, - FilterLastNDetectionsPandasTrackDataset, PandasDetection, PandasTrack, PandasTrackDataset, diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index 5cf5fdc93..104fe4be2 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -51,6 +51,9 @@ from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_cli.cli_application import OtAnalyticsCliApplicationStarter +from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( + FilterByClassPandasTrackDataset, +) from OTAnalytics.plugin_datastore.polars_track_store import ( POLARS_TRACK_GEOMETRY_FACTORY, PolarsTrackDataset, @@ -67,7 +70,6 @@ ShapelyTrackGeometryDataset, ) from OTAnalytics.plugin_datastore.track_store import ( - FilterByClassPandasTrackDataset, PandasByMaxConfidence, PandasTrackDataset, ) diff --git a/tests/utils/builders/track_dataset_provider.py b/tests/utils/builders/track_dataset_provider.py index 14a1ee7c3..a4e4459a7 100644 --- a/tests/utils/builders/track_dataset_provider.py +++ b/tests/utils/builders/track_dataset_provider.py @@ -10,6 +10,9 @@ TrackDataset, TrackGeometryDataset, ) +from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( + FilterByClassPandasTrackDataset, +) from OTAnalytics.plugin_datastore.python_track_store import ( FilteredPythonTrackDataset, PythonTrackDataset, @@ -17,10 +20,7 @@ from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, ) -from OTAnalytics.plugin_datastore.track_store import ( - FilterByClassPandasTrackDataset, - PandasTrackDataset, -) +from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset def create_mock_geometry_dataset( From fabdccd59929ee407899d55bdc44f055b02a0523 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 16:22:33 +0200 Subject: [PATCH 082/128] Extract test code --- .../test_filter_pandas_track_store.py | 84 +++++++++++++++++++ .../plugin_datastore/test_track_store.py | 78 ----------------- 2 files changed, 84 insertions(+), 78 deletions(-) create mode 100644 tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py diff --git a/tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py b/tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py new file mode 100644 index 000000000..230cd660f --- /dev/null +++ b/tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py @@ -0,0 +1,84 @@ +from utils.assertions import assert_track_datasets_equal + +from OTAnalytics.domain.track import Track +from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY +from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( + FilterByIdPandasTrackDataset, + FilterLastNDetectionsPandasTrackDataset, +) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrack +from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset + + +class TestFilterByIdPandasTrackDataset: + def test_filter_by_id_all_included( + self, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + ) -> None: + expected = PandasTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + dataset = PandasTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + track_ids = [car_track.id.id, pedestrian_track.id.id] + target = FilterByIdPandasTrackDataset(dataset, track_ids) + + assert_track_datasets_equal(target, expected) + + def test_filter_by_id_one_matching( + self, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + bicycle_track: Track, + ) -> None: + expected = PandasTrackDataset.from_list( + [pedestrian_track], track_geometry_factory + ) + dataset = PandasTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + track_ids = [bicycle_track.id.id, pedestrian_track.id.id] + target = FilterByIdPandasTrackDataset(dataset, track_ids) + + assert_track_datasets_equal(target, expected) + + +class TestFilterLastNSegmentsPandasTrackDataset: + def test_filter_last_n_segments_all_included( + self, + track_geometry_factory: TRACK_GEOMETRY_FACTORY, + car_track_continuing: Track, + pedestrian_track: Track, + single_detection_track: Track, + ) -> None: + expected_car_track = create_expected_track(car_track_continuing) + expected_pedestrian_track = create_expected_track(pedestrian_track) + expected = PandasTrackDataset.from_list( + [expected_car_track, expected_pedestrian_track], track_geometry_factory + ) + last_n = 2 + + target = FilterLastNDetectionsPandasTrackDataset( + PandasTrackDataset.from_list( + [car_track_continuing, pedestrian_track, single_detection_track], + track_geometry_factory, + ), + last_n, + ) + + assert_track_datasets_equal(target, expected) + + +def create_expected_track(track: Track) -> PythonTrack: + car_detections = track.detections[-2:] + expected_car_track = PythonTrack( + _original_id=track.id, + _id=track.id, + _classification=track.classification, + _detections=car_detections, + ) + return expected_car_track diff --git a/tests/OTAnalytics/plugin_datastore/test_track_store.py b/tests/OTAnalytics/plugin_datastore/test_track_store.py index 38ff29dd9..55096b32b 100644 --- a/tests/OTAnalytics/plugin_datastore/test_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_track_store.py @@ -17,10 +17,6 @@ TrackGeometryDataset, TrackIdSet, ) -from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( - FilterByIdPandasTrackDataset, - FilterLastNDetectionsPandasTrackDataset, -) from OTAnalytics.plugin_datastore.python_track_store import ( PythonTrack, PythonTrackDataset, @@ -670,77 +666,3 @@ def test_initializing_empty_dataset_has_correct_columns_index(self) -> None: assert actual.index.names == INDEX_NAMES assert INDEX_NAMES not in actual.columns.to_list() assert set(COLUMNS) - set(INDEX_NAMES) == set(actual.columns.to_list()) - - -class TestFilterByIdPandasTrackDataset: - def test_filter_by_id_all_included( - self, - track_geometry_factory: TRACK_GEOMETRY_FACTORY, - car_track: Track, - pedestrian_track: Track, - ) -> None: - expected = PandasTrackDataset.from_list( - [car_track, pedestrian_track], track_geometry_factory - ) - dataset = PandasTrackDataset.from_list( - [car_track, pedestrian_track], track_geometry_factory - ) - track_ids = [car_track.id.id, pedestrian_track.id.id] - target = FilterByIdPandasTrackDataset(dataset, track_ids) - - assert_track_datasets_equal(target, expected) - - def test_filter_by_id_one_matching( - self, - track_geometry_factory: TRACK_GEOMETRY_FACTORY, - car_track: Track, - pedestrian_track: Track, - bicycle_track: Track, - ) -> None: - expected = PandasTrackDataset.from_list( - [pedestrian_track], track_geometry_factory - ) - dataset = PandasTrackDataset.from_list( - [car_track, pedestrian_track], track_geometry_factory - ) - track_ids = [bicycle_track.id.id, pedestrian_track.id.id] - target = FilterByIdPandasTrackDataset(dataset, track_ids) - - assert_track_datasets_equal(target, expected) - - -class TestFilterLastNSegmentsPandasTrackDataset: - def test_filter_last_n_segments_all_included( - self, - track_geometry_factory: TRACK_GEOMETRY_FACTORY, - car_track_continuing: Track, - pedestrian_track: Track, - single_detection_track: Track, - ) -> None: - expected_car_track = create_expected_track(car_track_continuing) - expected_pedestrian_track = create_expected_track(pedestrian_track) - expected = PandasTrackDataset.from_list( - [expected_car_track, expected_pedestrian_track], track_geometry_factory - ) - last_n = 2 - - target = FilterLastNDetectionsPandasTrackDataset( - PandasTrackDataset.from_list( - [car_track_continuing, pedestrian_track, single_detection_track], - track_geometry_factory, - ), - last_n, - ) - - assert_track_datasets_equal(target, expected) - - -def create_expected_track(track: Track) -> PythonTrack: - car_detections = track.detections[-2:] - expected_car_track = PythonTrack( - _original_id=track.id, - _id=track.id, - _classification=track.classification, - _detections=car_detections, - ) - return expected_car_track From bcc5bb8722b45da2f819edfc3a23fea659c70006 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 16:38:55 +0200 Subject: [PATCH 083/128] Add filtered versions of TrackDataset for polars --- .../filter_polars_track_dataset.py | 322 ++++++++++++++++++ .../plugin_datastore/polars_track_store.py | 14 +- .../test_filter_polars_track_store.py | 95 ++++++ 3 files changed, 428 insertions(+), 3 deletions(-) create mode 100644 OTAnalytics/plugin_datastore/filter_polars_track_dataset.py create mode 100644 tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py diff --git a/OTAnalytics/plugin_datastore/filter_polars_track_dataset.py b/OTAnalytics/plugin_datastore/filter_polars_track_dataset.py new file mode 100644 index 000000000..5d1d39e38 --- /dev/null +++ b/OTAnalytics/plugin_datastore/filter_polars_track_dataset.py @@ -0,0 +1,322 @@ +from abc import abstractmethod +from typing import Iterable, Sequence + +import polars as pl + +from OTAnalytics.application.logger import logger +from OTAnalytics.domain import track +from OTAnalytics.domain.geometry import RelativeOffsetCoordinate +from OTAnalytics.domain.section import Section +from OTAnalytics.domain.track import Track, TrackId +from OTAnalytics.domain.track_dataset.filtered_track_dataset import ( + FilterByClassTrackDataset, + FilteredTrackDataset, +) +from OTAnalytics.domain.track_dataset.track_dataset import TrackIdSet +from OTAnalytics.plugin_datastore.polars_track_store import ( + POLARS_TRACK_GEOMETRY_FACTORY, + PolarsDataFrameProvider, + PolarsTrackClassificationCalculator, + PolarsTrackDataset, + get_rows_by_track_ids, +) +from OTAnalytics.plugin_datastore.track_store import MINIMUM_DETECTIONS, RANK + + +class FilteredPolarsTrackDataset( + FilteredTrackDataset, PolarsTrackDataset, PolarsDataFrameProvider +): + + @property + def track_geometry_factory(self) -> POLARS_TRACK_GEOMETRY_FACTORY: + return self._other.track_geometry_factory + + @property + def calculator(self) -> PolarsTrackClassificationCalculator: + return self._other.calculator + + def __init__(self, other: PolarsTrackDataset) -> None: + self._other = other + + def add_all(self, other: Iterable[Track]) -> PolarsTrackDataset: + return self.wrap(self._other.add_all(other)) + + def remove(self, track_id: TrackId) -> PolarsTrackDataset: + return self.wrap(self._other.remove(track_id)) + + def remove_multiple(self, track_ids: TrackIdSet) -> PolarsTrackDataset: + return self.wrap(self._other.remove_multiple(track_ids)) + + def clear(self) -> PolarsTrackDataset: + return self.wrap(self._other.clear()) + + def split(self, chunks: int) -> Sequence[PolarsTrackDataset]: + return [self.wrap(dataset) for dataset in self._other.split(chunks)] + + def calculate_geometries_for( + self, offsets: Iterable[RelativeOffsetCoordinate] + ) -> None: + self._other.calculate_geometries_for(offsets) + + def cut_with_section( + self, section: Section, offset: RelativeOffsetCoordinate + ) -> tuple[PolarsTrackDataset, TrackIdSet]: + dataset, original_track_ids = self._other.cut_with_section(section, offset) + return self.wrap(dataset), original_track_ids + + @abstractmethod + def _filter(self) -> PolarsTrackDataset: + raise NotImplementedError + + @abstractmethod + def wrap(self, other: PolarsTrackDataset) -> PolarsTrackDataset: + raise NotImplementedError + + def get_data(self) -> pl.DataFrame: + return self._filter().get_data() + + def revert_cuts_for( + self, original_track_ids: TrackIdSet + ) -> tuple[PolarsTrackDataset, TrackIdSet, TrackIdSet]: + reverted_dataset, reverted_ids, cut_ids = self._other.revert_cuts_for( + original_track_ids + ) + return self.wrap(reverted_dataset), reverted_ids, cut_ids + + def remove_by_original_ids( + self, original_ids: TrackIdSet + ) -> tuple["PolarsTrackDataset", TrackIdSet]: + updated_dataset, removed_ids = self._other.remove_by_original_ids(original_ids) + return self.wrap(updated_dataset), removed_ids + + +class FilterByClassPolarsTrackDataset( + FilteredPolarsTrackDataset, FilterByClassTrackDataset +): + @property + def include_classes(self) -> frozenset[str]: + return self._include_classes + + @property + def exclude_classes(self) -> frozenset[str]: + return self._exclude_classes + + def __init__( + self, + other: PolarsTrackDataset, + include_classes: frozenset[str], + exclude_classes: frozenset[str], + ) -> None: + super().__init__(other) + self._include_classes = include_classes + self._exclude_classes = exclude_classes + self._cache: PolarsTrackDataset | None = None + + def _filter(self) -> PolarsTrackDataset: + """Filter TrackDataset by classifications. + + IMPORTANT: Classifications contained in the include_classes will not be + removed even if they appear in the set of exclude_classes. + Furthermore, the whitelist will not be applied if empty. + + Returns: + TrackDataset: the filtered dataset. + """ + if not self.include_classes and not self._exclude_classes: + return self._other + + if self._cache is not None: + return self._cache + + if self.include_classes: + logger().info( + "Apply 'include-classes' filter to filter tracks: " + f"{self.include_classes}" + "\n'exclude-classes' filter is not used" + ) + filtered_dataset = self._get_dataset_with_classes( + list(self._other.classifications & self.include_classes) + ) + elif self.exclude_classes: + logger().info( + "Apply 'exclude-classes' filter to filter tracks: " + f"{self.exclude_classes}" + ) + filtered_dataset = self._get_dataset_with_classes( + list(self._other.classifications - self.exclude_classes) + ) + else: + return self._other + self._cache = filtered_dataset + return filtered_dataset + + def _get_dataset_with_classes(self, classes: list[str]) -> PolarsTrackDataset: + if self._other.empty: + return self._other + dataset = self._other.get_data() + mask = dataset[track.TRACK_CLASSIFICATION].is_in(classes) + filtered_df = dataset.filter(mask) + # The polars DataFrame filtering approach ensures data consistency. + # When filtering by track classes, we invalidate the entire geometry cache + # rather than attempting selective removal. This approach is acceptable + # because track removal only occurs when cutting tracks, which is a rare use + # case. + + return PolarsTrackDataset( + track_geometry_factory=self._other.track_geometry_factory, + dataset=filtered_df, + geometry_datasets=None, + calculator=self._other.calculator, + ) + + def wrap(self, other: PolarsTrackDataset) -> PolarsTrackDataset: + return FilterByClassPolarsTrackDataset( + other, self.include_classes, self.exclude_classes + ) + + +class FilterByIdPolarsTrackDataset(FilteredPolarsTrackDataset): + """ + Represents a dataset that filters tracks by specific track IDs. + + This class provides functionality to filter a Polars-based track dataset by a + list of track IDs. It utilizes caching to optimize repeated filtering operations, + manages dataset consistency, and ensures efficient geometry cache invalidation + for cases such as track removal. + """ + + def __init__(self, other: PolarsTrackDataset, track_ids: list[str]) -> None: + super().__init__(other) + self._included_track_ids = track_ids + self._cache: PolarsTrackDataset | None = None + + def _filter(self) -> PolarsTrackDataset: + """ + Filters the track dataset based on included track IDs. + + This method applies a filter on a dataset to include only the tracks + specified in `_included_track_ids`. If there are no specified track IDs, + it returns the original dataset (`_other`). It employs caching to store the + filtered dataset for future use. + + Returns: + TrackDataset: The filtered dataset containing only the specified track IDs. + If no track IDs are provided, the original dataset is returned. + """ + if not self._included_track_ids: + return self._other + + if self._cache is not None: + return self._cache + + if self._included_track_ids: + logger().info( + "Apply 'track-ids' filter to filter tracks: " + f"{self._included_track_ids}" + ) + filtered_dataset = self._get_dataset_with( + track_ids=self._included_track_ids + ) + else: + return self._other + self._cache = filtered_dataset + return filtered_dataset + + def _get_dataset_with(self, track_ids: list[str]) -> PolarsTrackDataset: + if self._other.empty: + return self._other + dataset = self._other.get_data() + filtered_df = get_rows_by_track_ids(dataset, track_ids) + # The polars DataFrame filtering approach ensures data consistency. + # When filtering by track IDs, we invalidate the entire geometry cache + # rather than attempting selective removal. This approach is acceptable + # because track removal only occurs when cutting tracks, which is a rare use + # case. + + return PolarsTrackDataset( + track_geometry_factory=self._other.track_geometry_factory, + dataset=filtered_df, + geometry_datasets=None, + calculator=self._other.calculator, + ) + + def wrap(self, other: PolarsTrackDataset) -> PolarsTrackDataset: + return FilterByIdPolarsTrackDataset(other, self._included_track_ids) + + +class FilterLastNDetectionsPolarsTrackDataset(FilteredPolarsTrackDataset): + def __init__(self, other: PolarsTrackDataset, n: int) -> None: + super().__init__(other) + self._n = n + self._cache: PolarsTrackDataset | None = None + + def _filter(self) -> PolarsTrackDataset: + """ + Filters the track dataset based on included track IDs. + + This method applies a filter on a dataset to include only the tracks + specified in `_included_track_ids`. If there are no specified track IDs, + it returns the original dataset (`_other`). It employs caching to store the + filtered dataset for future use. + + Returns: + TrackDataset: The filtered dataset containing only the specified track IDs. + If no track IDs are provided, the original dataset is returned. + """ + if self._cache is not None: + return self._cache + + logger().info( + f"Limit number of detections per track to last {self._n} of them." + ) + filtered_dataset = self._get_filtered_dataset() + self._cache = filtered_dataset + return filtered_dataset + + def _get_filtered_dataset(self) -> PolarsTrackDataset: + if self._other.empty: + return self._other + dataset = self._other.get_data() + filtered_df = get_exactly_two_latest_occurrences_per_id(dataset, self._n) + # The polars DataFrame filtering approach ensures data consistency. + # When filtering by number of detections, we invalidate the entire geometry + # cache rather than attempting selective removal. This approach is acceptable + # because track removal only occurs when cutting tracks, which is a rare use + # case. + + return PolarsTrackDataset( + track_geometry_factory=self._other.track_geometry_factory, + dataset=filtered_df, + geometry_datasets=None, + calculator=self._other.calculator, + ) + + def wrap(self, other: PolarsTrackDataset) -> PolarsTrackDataset: + return FilterLastNDetectionsPolarsTrackDataset(other, self._n) + + +def get_latest_occurrences(dataframe: pl.DataFrame, last_n: int) -> pl.DataFrame: + result = dataframe.with_columns( + [ + pl.col(track.OCCURRENCE) + .rank("ordinal", descending=True) + .over(track.TRACK_ID) + .alias(RANK) + ] + ) + return result.filter(pl.col(RANK) <= last_n).drop(RANK) + + +def get_exactly_two_latest_occurrences_per_id( + dataframe: pl.DataFrame, last_n: int +) -> pl.DataFrame: + counts = dataframe.group_by(track.TRACK_ID).agg(pl.len().alias("count")) + valid_ids = ( + counts.filter(pl.col("count") >= MINIMUM_DETECTIONS) + .select(track.TRACK_ID) + .to_series() + ) + filtered_df = get_rows_by_track_ids( + dataframe=dataframe, track_ids=valid_ids.to_list() + ) + return get_latest_occurrences(filtered_df, last_n=last_n) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 85d628817..a771dcb84 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -304,6 +304,14 @@ def classifications(self) -> frozenset[str]: def empty(self) -> bool: return self._dataset.is_empty() + @property + def track_geometry_factory(self) -> POLARS_TRACK_GEOMETRY_FACTORY: + return self._track_geometry_factory + + @property + def calculator(self) -> PolarsTrackClassificationCalculator: + return self._calculator + def __init__( self, track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, @@ -318,8 +326,8 @@ def __init__( else: self._dataset = create_empty_dataframe() - self.calculator = calculator - self.track_geometry_factory = track_geometry_factory + self._calculator = calculator + self._track_geometry_factory = track_geometry_factory if geometry_datasets is None: self._geometry_datasets = dict[ RelativeOffsetCoordinate, PolarsTrackGeometryDataset @@ -630,7 +638,7 @@ def __create_segments(self) -> pl.DataFrame: def cut_with_section( self, section: Section, offset: RelativeOffsetCoordinate - ) -> tuple["TrackDataset", TrackIdSet]: + ) -> tuple["PolarsTrackDataset", TrackIdSet]: if len(self) == 0: logger().info("No tracks to cut") from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet diff --git a/tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py b/tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py new file mode 100644 index 000000000..b9b57c80f --- /dev/null +++ b/tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py @@ -0,0 +1,95 @@ +import pytest +from utils.assertions import assert_track_datasets_equal + +from OTAnalytics.domain.track import Track +from OTAnalytics.plugin_datastore.filter_polars_track_dataset import ( + FilterByIdPolarsTrackDataset, + FilterLastNDetectionsPolarsTrackDataset, +) +from OTAnalytics.plugin_datastore.polars_track_store import ( + POLARS_TRACK_GEOMETRY_FACTORY, + PolarsTrackDataset, +) +from OTAnalytics.plugin_datastore.python_track_store import PythonTrack +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + PolarsTrackGeometryDataset, +) + + +@pytest.fixture +def track_geometry_factory() -> POLARS_TRACK_GEOMETRY_FACTORY: + return PolarsTrackGeometryDataset.from_track_dataset + + +class TestFilterByIdPolarsTrackDataset: + def test_filter_by_id_all_included( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + ) -> None: + expected = PolarsTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + dataset = PolarsTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + track_ids = [car_track.id.id, pedestrian_track.id.id] + target = FilterByIdPolarsTrackDataset(dataset, track_ids) + + assert_track_datasets_equal(target, expected) + + def test_filter_by_id_one_matching( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + car_track: Track, + pedestrian_track: Track, + bicycle_track: Track, + ) -> None: + expected = PolarsTrackDataset.from_list( + [pedestrian_track], track_geometry_factory + ) + dataset = PolarsTrackDataset.from_list( + [car_track, pedestrian_track], track_geometry_factory + ) + track_ids = [bicycle_track.id.id, pedestrian_track.id.id] + target = FilterByIdPolarsTrackDataset(dataset, track_ids) + + assert_track_datasets_equal(target, expected) + + +class TestFilterLastNSegmentsPolarsTrackDataset: + def test_filter_last_n_segments_all_included( + self, + track_geometry_factory: POLARS_TRACK_GEOMETRY_FACTORY, + car_track_continuing: Track, + pedestrian_track: Track, + single_detection_track: Track, + ) -> None: + expected_car_track = create_expected_track(car_track_continuing) + expected_pedestrian_track = create_expected_track(pedestrian_track) + expected = PolarsTrackDataset.from_list( + [expected_car_track, expected_pedestrian_track], track_geometry_factory + ) + last_n = 2 + + target = FilterLastNDetectionsPolarsTrackDataset( + PolarsTrackDataset.from_list( + [car_track_continuing, pedestrian_track, single_detection_track], + track_geometry_factory, + ), + last_n, + ) + + assert_track_datasets_equal(target, expected) + + +def create_expected_track(track: Track) -> PythonTrack: + car_detections = track.detections[-2:] + expected_car_track = PythonTrack( + _original_id=track.id, + _id=track.id, + _classification=track.classification, + _detections=car_detections, + ) + return expected_car_track From 4ddd992c39fc4d005e352c9fe6056cf0d7ab5f53 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 16:43:06 +0200 Subject: [PATCH 084/128] Add script to check clean architecture violations --- scripts/check_clean_architecture.py | 292 ++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 scripts/check_clean_architecture.py diff --git a/scripts/check_clean_architecture.py b/scripts/check_clean_architecture.py new file mode 100644 index 000000000..03bb96376 --- /dev/null +++ b/scripts/check_clean_architecture.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +""" +Clean Architecture Import Checker for OTAnalytics + +This script validates that the OTAnalytics project follows Clean Architecture import +rules. The architecture has 4 layers: +1. domain (innermost layer) +2. application +3. adapter_* +4. plugin_* (outermost layer) + +Import Rules: +- Modules can only import from the same layer or inner (more central) layers +- Domain can only import from domain and standard libraries +- Application can import from domain, application, and standard libraries +- Adapter can import from domain, application, adapter, and standard libraries +- Plugin can import from all layers and standard libraries +- NO imports from outer layers to inner layers are allowed + +Usage: + python check_clean_architecture.py [--path PATH] [--verbose] +""" + +import ast +import os +import sys +from dataclasses import dataclass +from enum import Enum, auto +from pathlib import Path +from typing import Dict, List, Tuple + + +class ArchitecturalLayer(Enum): + """Defines the architectural layers in order from innermost to outermost""" + + DOMAIN = auto() + APPLICATION = auto() + ADAPTER = auto() + PLUGIN = auto() + HELPERS = auto() # Utility layer + EXTERNAL = auto() # Standard library and third-party + + +@dataclass +class ImportViolation: + """Represents a Clean Architecture import violation""" + + file_path: str + line_number: int + imported_module: str + violating_import: str + source_layer: ArchitecturalLayer + target_layer: ArchitecturalLayer + reason: str + + +class CleanArchitectureChecker: + def __init__(self, project_root: str = "OTAnalytics"): + self.project_root = Path(project_root) + self.violations: List[ImportViolation] = [] + + # Define layer hierarchy (inner to outer) + self.layer_hierarchy = [ + ArchitecturalLayer.DOMAIN, + ArchitecturalLayer.APPLICATION, + ArchitecturalLayer.ADAPTER, + ArchitecturalLayer.PLUGIN, + ArchitecturalLayer.HELPERS, + ] + + def categorize_module_layer(self, module_name: str) -> ArchitecturalLayer: + """Determine which architectural layer a module belongs to""" + if not module_name.startswith("OTAnalytics"): + return ArchitecturalLayer.EXTERNAL + + parts = module_name.split(".") + if len(parts) < 2: + return ArchitecturalLayer.EXTERNAL + + layer_part = parts[1] # OTAnalytics. + + if layer_part == "domain": + return ArchitecturalLayer.DOMAIN + elif layer_part == "application": + return ArchitecturalLayer.APPLICATION + elif layer_part.startswith("adapter_"): + return ArchitecturalLayer.ADAPTER + elif layer_part.startswith("plugin_"): + return ArchitecturalLayer.PLUGIN + elif layer_part == "helpers": + return ArchitecturalLayer.HELPERS + else: + # Unknown OTAnalytics module, treat as external + return ArchitecturalLayer.EXTERNAL + + def is_import_allowed( + self, source_layer: ArchitecturalLayer, target_layer: ArchitecturalLayer + ) -> bool: + """Check if importing from target_layer to source_layer is allowed""" + if target_layer == ArchitecturalLayer.EXTERNAL: + return True # All layers can import external modules + + if target_layer == ArchitecturalLayer.HELPERS: + return True # All layers can import from helpers (utility layer) + + if source_layer == target_layer: + return True # Same layer imports are allowed + + try: + source_index = self.layer_hierarchy.index(source_layer) + target_index = self.layer_hierarchy.index(target_layer) + + # Can only import from same layer or inner layers (lower index) + return target_index <= source_index + except ValueError: + # If layer not found in hierarchy, be permissive + return True + + def extract_imports_from_file(self, file_path: Path) -> List[Tuple[int, str]]: + """Extract all import statements from a Python file""" + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + + tree = ast.parse(content, filename=str(file_path)) + imports = [] + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + imports.append((node.lineno, alias.name)) + elif isinstance(node, ast.ImportFrom): + if node.module: + imports.append((node.lineno, node.module)) + + return imports + + except (SyntaxError, UnicodeDecodeError, Exception) as e: + print(f"Warning: Could not parse {file_path}: {e}") + return [] + + def get_module_layer_from_path(self, file_path: Path) -> ArchitecturalLayer: + """Determine the architectural layer based on the file path""" + relative_path = file_path.relative_to(self.project_root) + parts = relative_path.parts + + if len(parts) == 0: + return ArchitecturalLayer.EXTERNAL + + first_part = parts[0] + + if first_part == "domain": + return ArchitecturalLayer.DOMAIN + elif first_part == "application": + return ArchitecturalLayer.APPLICATION + elif first_part.startswith("adapter_"): + return ArchitecturalLayer.ADAPTER + elif first_part.startswith("plugin_"): + return ArchitecturalLayer.PLUGIN + elif first_part == "helpers": + return ArchitecturalLayer.HELPERS + else: + return ArchitecturalLayer.EXTERNAL + + def check_file(self, file_path: Path) -> None: + """Check a single Python file for import violations""" + if not file_path.suffix == ".py": + return + + source_layer = self.get_module_layer_from_path(file_path) + if source_layer == ArchitecturalLayer.EXTERNAL: + return # Skip files not in our architecture + + imports = self.extract_imports_from_file(file_path) + + for line_number, imported_module in imports: + target_layer = self.categorize_module_layer(imported_module) + + if not self.is_import_allowed(source_layer, target_layer): + violation = ImportViolation( + file_path=str(file_path.relative_to(self.project_root)), + line_number=line_number, + imported_module=imported_module, + violating_import=imported_module, + source_layer=source_layer, + target_layer=target_layer, + reason=f"{source_layer.name} layer cannot import from " + f"{target_layer.name} layer", + ) + self.violations.append(violation) + + def check_directory(self, directory: Path) -> None: + """Recursively check all Python files in a directory""" + for root, dirs, files in os.walk(directory): + # Skip __pycache__ directories + dirs[:] = [d for d in dirs if d != "__pycache__"] + + for file in files: + if file.endswith(".py"): + file_path = Path(root) / file + self.check_file(file_path) + + def run_check(self) -> None: + """Run the complete clean architecture check""" + if not self.project_root.exists(): + print(f"Error: Project root '{self.project_root}' does not exist") + return + + print(f"Checking Clean Architecture compliance in {self.project_root}") + print( + "Architecture layers (innermost → outermost): " + f"{' → '.join(layer.name for layer in self.layer_hierarchy)}" + ) + print() + + self.check_directory(self.project_root) + + def print_results(self, verbose: bool = False) -> None: + """Print the results of the architecture check""" + if not self.violations: + print("✅ No Clean Architecture violations found!") + return + + print(f"❌ Found {len(self.violations)} Clean Architecture violations:") + print() + + # Group violations by file + violations_by_file: dict[str, List[ImportViolation]] = {} + for violation in self.violations: + if violation.file_path not in violations_by_file: + violations_by_file[violation.file_path] = [] + violations_by_file[violation.file_path].append(violation) + + for file_path in sorted(violations_by_file.keys()): + print(f"📄 {file_path}:") + for violation in violations_by_file[file_path]: + print(f" Line {violation.line_number}: {violation.imported_module}") + print(f" ❌ {violation.reason}") + if verbose: + print( + f" Source: {violation.source_layer.name}, " + f"Target: {violation.target_layer.name}" + ) + print() + + def get_summary(self) -> Dict[str, int]: + """Get a summary of violations by layer""" + summary: dict[str, int] = {} + for violation in self.violations: + key = f"{violation.source_layer.name} → {violation.target_layer.name}" + summary[key] = summary.get(key, 0) + 1 + return summary + + +def main() -> None: + import argparse + + parser = argparse.ArgumentParser( + description="Check Clean Architecture compliance for OTAnalytics" + ) + parser.add_argument( + "--path", + default="OTAnalytics", + help="Path to OTAnalytics project directory", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Show detailed violation information", + ) + + args = parser.parse_args() + + checker = CleanArchitectureChecker(args.path) + checker.run_check() + checker.print_results(verbose=args.verbose) + + if checker.violations: + print("\n📊 Violation Summary:") + summary = checker.get_summary() + for violation_type, count in sorted(summary.items()): + print(f" {violation_type}: {count}") + + print(f"\nTotal violations: {len(checker.violations)}") + sys.exit(1) + else: + print("🎉 All imports follow Clean Architecture principles!") + sys.exit(0) + + +if __name__ == "__main__": + main() From 51c2b05672a28e1cac12eff229c4e3fddb1ec652 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 16:46:16 +0200 Subject: [PATCH 085/128] Fix imports --- .../plugin_datastore/test_filter_pandas_track_store.py | 3 +-- .../plugin_datastore/test_filter_polars_track_store.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py b/tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py index 230cd660f..988e15b59 100644 --- a/tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_filter_pandas_track_store.py @@ -1,5 +1,3 @@ -from utils.assertions import assert_track_datasets_equal - from OTAnalytics.domain.track import Track from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( @@ -8,6 +6,7 @@ ) from OTAnalytics.plugin_datastore.python_track_store import PythonTrack from OTAnalytics.plugin_datastore.track_store import PandasTrackDataset +from tests.utils.assertions import assert_track_datasets_equal class TestFilterByIdPandasTrackDataset: diff --git a/tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py b/tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py index b9b57c80f..cba5ec664 100644 --- a/tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_filter_polars_track_store.py @@ -1,5 +1,4 @@ import pytest -from utils.assertions import assert_track_datasets_equal from OTAnalytics.domain.track import Track from OTAnalytics.plugin_datastore.filter_polars_track_dataset import ( @@ -14,6 +13,7 @@ from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( PolarsTrackGeometryDataset, ) +from tests.utils.assertions import assert_track_datasets_equal @pytest.fixture From ba1ceb9f435fe01fb871875be63b0f9dc95d2da8 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 17:08:08 +0200 Subject: [PATCH 086/128] Activate filter classes using polars --- .../track_dataset/filtered_track_dataset.py | 6 ++++++ OTAnalytics/plugin_ui/base_application.py | 21 +++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py index 6c781f928..6751cf504 100644 --- a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py +++ b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py @@ -7,6 +7,7 @@ from OTAnalytics.domain.track import Track, TrackId from OTAnalytics.domain.track_dataset.track_dataset import ( IntersectionPoint, + IntersectionPointsDataset, TrackDataset, TrackIdSet, TrackSegmentDataset, @@ -60,6 +61,11 @@ def intersection_points( ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: return self._filter().intersection_points(sections, offset) + def wrap_intersection_points( + self, sections: list[Section], offset: RelativeOffsetCoordinate + ) -> IntersectionPointsDataset: + return self._filter().wrap_intersection_points(sections, offset) + def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: diff --git a/OTAnalytics/plugin_ui/base_application.py b/OTAnalytics/plugin_ui/base_application.py index c6c7fc76d..85fb9a48b 100644 --- a/OTAnalytics/plugin_ui/base_application.py +++ b/OTAnalytics/plugin_ui/base_application.py @@ -187,6 +187,9 @@ from OTAnalytics.domain.track_id_provider import TrackIdProvider from OTAnalytics.domain.track_repository import TrackFileRepository, TrackRepository from OTAnalytics.domain.video import VideoRepository +from OTAnalytics.plugin_datastore.filter_polars_track_dataset import ( + FilterByClassPolarsTrackDataset, +) from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSetFactory from OTAnalytics.plugin_datastore.polars_track_store import ( POLARS_TRACK_GEOMETRY_FACTORY, @@ -603,15 +606,15 @@ def datastore(self) -> Datastore: @cached_property def track_repository(self) -> TrackRepository: return TrackRepository( - # FilterByClassPandasTrackDataset( - PolarsTrackDataset.from_list( - [], - self.track_geometry_factory, - self.polars_by_max_confidence, - ), - # self.run_config.include_classes, - # self.run_config.exclude_classes, - # ) + FilterByClassPolarsTrackDataset( + PolarsTrackDataset.from_list( + [], + self.track_geometry_factory, + self.polars_by_max_confidence, + ), + self.run_config.include_classes, + self.run_config.exclude_classes, + ) ) def _create_track_parser(self) -> TrackParser: From f161c22882dc8e22c26267b752dcd572c4e4a059 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 17:18:25 +0200 Subject: [PATCH 087/128] Remove unused wrapped method --- .../track_dataset/filtered_track_dataset.py | 6 ------ .../domain/track_dataset/track_dataset.py | 19 +------------------ .../plugin_datastore/polars_track_store.py | 7 ------- .../plugin_datastore/python_track_store.py | 10 ++++++---- OTAnalytics/plugin_datastore/track_store.py | 12 +++++++----- .../test_filtered_track_dataset.py | 6 ++++-- 6 files changed, 18 insertions(+), 42 deletions(-) diff --git a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py index 6751cf504..47ff626f6 100644 --- a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py +++ b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py @@ -6,7 +6,6 @@ from OTAnalytics.domain.section import Section, SectionId from OTAnalytics.domain.track import Track, TrackId from OTAnalytics.domain.track_dataset.track_dataset import ( - IntersectionPoint, IntersectionPointsDataset, TrackDataset, TrackIdSet, @@ -56,11 +55,6 @@ def intersecting_tracks( ) -> TrackIdSet: return self._filter().intersecting_tracks(sections, offset) - def intersection_points( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - return self._filter().intersection_points(sections, offset) - def wrap_intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> IntersectionPointsDataset: diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 55dc6186a..1454e2d0d 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -232,22 +232,6 @@ def intersecting_tracks( raise NotImplementedError @abstractmethod - def intersection_points( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - """ - Return the intersection points resulting from the tracks and the - given sections. - - Args: - sections (list[Section]): the sections to intersect with. - offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. - - Returns: - dict[TrackId, list[tuple[SectionId]]]: the intersection points. - """ - raise NotImplementedError - def wrap_intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> "IntersectionPointsDataset": @@ -261,8 +245,7 @@ def wrap_intersection_points( Returns: IntersectionPointsDataset: the intersection points dataset. """ - intersection_data = self.intersection_points(sections, offset) - return PythonIntersectionPointsDataset(intersection_data, self) + raise NotImplementedError @abstractmethod def contained_by_sections( diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index a771dcb84..1ed87e181 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -38,7 +38,6 @@ START_X, START_Y, EmptyTrackIdSet, - IntersectionPoint, IntersectionPointsDataset, TrackDataset, TrackDoesNotExistError, @@ -490,12 +489,6 @@ def _get_geometry_dataset_for( self._geometry_datasets[offset] = geometry_dataset return geometry_dataset - def intersection_points( - self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: - geometry_dataset = self._get_geometry_dataset_for(offset) - return geometry_dataset.intersection_points(sections) - def wrap_intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> IntersectionPointsDataset: diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 1d665b60b..95eca8eab 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -37,7 +37,8 @@ START_Y, TRACK_GEOMETRY_FACTORY, EmptyTrackIdSet, - IntersectionPoint, + IntersectionPointsDataset, + PythonIntersectionPointsDataset, TrackDataset, TrackDoesNotExistError, TrackGeometryDataset, @@ -613,11 +614,12 @@ def _get_geometry_dataset_for( self._geometry_datasets[offset] = geometry_dataset return geometry_dataset - def intersection_points( + def wrap_intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + ) -> IntersectionPointsDataset: geometry_dataset = self._get_geometry_dataset_for(offset) - return geometry_dataset.intersection_points(sections) + intersection_data = geometry_dataset.intersection_points(sections) + return PythonIntersectionPointsDataset(intersection_data, self) def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 189a04278..00e36b525 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -29,7 +29,8 @@ START_Y, TRACK_GEOMETRY_FACTORY, EmptyTrackIdSet, - IntersectionPoint, + IntersectionPointsDataset, + PythonIntersectionPointsDataset, TrackDataset, TrackDoesNotExistError, TrackGeometryDataset, @@ -526,11 +527,12 @@ def _get_geometry_dataset_for( self._geometry_datasets[offset] = geometry_dataset return geometry_dataset - def intersection_points( + def wrap_intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate - ) -> dict[TrackId, list[tuple[SectionId, IntersectionPoint]]]: + ) -> IntersectionPointsDataset: geometry_dataset = self._get_geometry_dataset_for(offset) - return geometry_dataset.intersection_points(sections) + intersection_data = geometry_dataset.intersection_points(sections) + return PythonIntersectionPointsDataset(intersection_data, self) def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate @@ -633,7 +635,7 @@ def cut_with_section( # - calculate intersects per segment # - cumcount per intersects by track id # - add cumcount to track id - intersection_points = self.intersection_points([section], offset) + intersection_points = self.wrap_intersection_points([section], offset) cut_indices = { unpack(track_id): [ ip[1].upper_index diff --git a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py index f87c06fd5..8632335ec 100644 --- a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py +++ b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py @@ -246,8 +246,10 @@ def test_intersection_points(self) -> None: sections = Mock() offset = Mock() for dataset, mock_other in mocked_datasets: - dataset.intersection_points(sections, offset) - mock_other.intersection_points.assert_called_once_with(sections, offset) + dataset.wrap_intersection_points(sections, offset) + mock_other.wrap_intersection_points.assert_called_once_with( + sections, offset + ) def test_contained_by_section(self) -> None: mocked_datasets = self.get_mocked_datasets([], []) From 94389fe01f934a02fe766f11bc2f17248921e042 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 17:21:48 +0200 Subject: [PATCH 088/128] Renamed wrapped method name --- .../use_cases/create_intersection_events.py | 2 +- .../track_dataset/filtered_track_dataset.py | 4 +-- .../domain/track_dataset/track_dataset.py | 2 +- .../plugin_datastore/polars_track_store.py | 2 +- .../plugin_datastore/python_track_store.py | 2 +- OTAnalytics/plugin_datastore/track_store.py | 4 +-- .../test_create_intersection_events.py | 28 +++++++++---------- .../test_filtered_track_dataset.py | 6 ++-- 8 files changed, 23 insertions(+), 27 deletions(-) diff --git a/OTAnalytics/application/use_cases/create_intersection_events.py b/OTAnalytics/application/use_cases/create_intersection_events.py index 3bb14219c..5b607d2cc 100644 --- a/OTAnalytics/application/use_cases/create_intersection_events.py +++ b/OTAnalytics/application/use_cases/create_intersection_events.py @@ -62,7 +62,7 @@ def __do_intersect( offset: RelativeOffsetCoordinate, event_builder: EventBuilder, ) -> EventDataset: - intersection_result = track_dataset.wrap_intersection_points(sections, offset) + intersection_result = track_dataset.intersection_points(sections, offset) return intersection_result.create_events(offset, event_builder) diff --git a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py index 47ff626f6..55b3643c8 100644 --- a/OTAnalytics/domain/track_dataset/filtered_track_dataset.py +++ b/OTAnalytics/domain/track_dataset/filtered_track_dataset.py @@ -55,10 +55,10 @@ def intersecting_tracks( ) -> TrackIdSet: return self._filter().intersecting_tracks(sections, offset) - def wrap_intersection_points( + def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> IntersectionPointsDataset: - return self._filter().wrap_intersection_points(sections, offset) + return self._filter().intersection_points(sections, offset) def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate diff --git a/OTAnalytics/domain/track_dataset/track_dataset.py b/OTAnalytics/domain/track_dataset/track_dataset.py index 1454e2d0d..255df5115 100644 --- a/OTAnalytics/domain/track_dataset/track_dataset.py +++ b/OTAnalytics/domain/track_dataset/track_dataset.py @@ -232,7 +232,7 @@ def intersecting_tracks( raise NotImplementedError @abstractmethod - def wrap_intersection_points( + def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> "IntersectionPointsDataset": """ diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 1ed87e181..3e8d6b836 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -489,7 +489,7 @@ def _get_geometry_dataset_for( self._geometry_datasets[offset] = geometry_dataset return geometry_dataset - def wrap_intersection_points( + def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> IntersectionPointsDataset: geometry_dataset = self._get_geometry_dataset_for(offset) diff --git a/OTAnalytics/plugin_datastore/python_track_store.py b/OTAnalytics/plugin_datastore/python_track_store.py index 95eca8eab..487d3ce98 100644 --- a/OTAnalytics/plugin_datastore/python_track_store.py +++ b/OTAnalytics/plugin_datastore/python_track_store.py @@ -614,7 +614,7 @@ def _get_geometry_dataset_for( self._geometry_datasets[offset] = geometry_dataset return geometry_dataset - def wrap_intersection_points( + def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> IntersectionPointsDataset: geometry_dataset = self._get_geometry_dataset_for(offset) diff --git a/OTAnalytics/plugin_datastore/track_store.py b/OTAnalytics/plugin_datastore/track_store.py index 00e36b525..d938e8c3e 100644 --- a/OTAnalytics/plugin_datastore/track_store.py +++ b/OTAnalytics/plugin_datastore/track_store.py @@ -527,7 +527,7 @@ def _get_geometry_dataset_for( self._geometry_datasets[offset] = geometry_dataset return geometry_dataset - def wrap_intersection_points( + def intersection_points( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> IntersectionPointsDataset: geometry_dataset = self._get_geometry_dataset_for(offset) @@ -635,7 +635,7 @@ def cut_with_section( # - calculate intersects per segment # - cumcount per intersects by track id # - add cumcount to track id - intersection_points = self.wrap_intersection_points([section], offset) + intersection_points = self.intersection_points([section], offset) cut_indices = { unpack(track_id): [ ip[1].upper_index diff --git a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py index affb01043..4440d3d7c 100644 --- a/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py +++ b/tests/OTAnalytics/application/use_cases/test_create_intersection_events.py @@ -131,7 +131,7 @@ def assert_valid(self, event_results: EventDataset, event_builder: Mock) -> None self._assert_valid(event_results, event_builder) def _assert_valid(self, event_results: EventDataset, event_builder: Mock) -> None: - self.track_dataset.wrap_intersection_points.assert_called_once_with( + self.track_dataset.intersection_points.assert_called_once_with( [self.section], self.section.get_offset(EventType.SECTION_ENTER) ) if self.expected_event_coords: @@ -250,8 +250,8 @@ def test_case_track_line_section(track: Track) -> _TestCase: prev_detection = track.detections[ip.lower_index] detection = track.detections[ip.upper_index] track_dataset = Mock(spec=TrackDataset) - track_dataset.wrap_intersection_points.return_value = ( - PythonIntersectionPointsDataset({track.id: [(section.id, ip)]}, track_dataset) + track_dataset.intersection_points.return_value = PythonIntersectionPointsDataset( + {track.id: [(section.id, ip)]}, track_dataset ) track_dataset.get_for.return_value = track expected_event_coord = _ExpectedEventCoord.from_detection( @@ -298,16 +298,14 @@ def test_case_closed_track_line_section( ) -> _TestCase: section = create_section([(0, 1.5), (3, 1.5)], SectionType.LINE) track_dataset = Mock(spec=TrackDataset) - track_dataset.wrap_intersection_points.return_value = ( - PythonIntersectionPointsDataset( - { - closed_track.id: [ - (section.id, IntersectionPoint(upper_index=2, relative_position=0)), - (section.id, IntersectionPoint(upper_index=4, relative_position=0)), - ] - }, - track_dataset, - ) + track_dataset.intersection_points.return_value = PythonIntersectionPointsDataset( + { + closed_track.id: [ + (section.id, IntersectionPoint(upper_index=2, relative_position=0)), + (section.id, IntersectionPoint(upper_index=4, relative_position=0)), + ] + }, + track_dataset, ) track_dataset.get_for.return_value = closed_track expected_event_coords = [ @@ -328,8 +326,8 @@ def test_case_closed_track_line_section( def test_case_line_section_no_intersection(track: Track) -> _TestCase: section = create_section([(0, 0), (10, 0)], SectionType.LINE) track_dataset = Mock(spec=TrackDataset) - track_dataset.wrap_intersection_points.return_value = ( - PythonIntersectionPointsDataset({}, track_dataset) + track_dataset.intersection_points.return_value = PythonIntersectionPointsDataset( + {}, track_dataset ) track_dataset.get_for.return_value = track diff --git a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py index 8632335ec..f87c06fd5 100644 --- a/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py +++ b/tests/OTAnalytics/domain/track_dataset/test_filtered_track_dataset.py @@ -246,10 +246,8 @@ def test_intersection_points(self) -> None: sections = Mock() offset = Mock() for dataset, mock_other in mocked_datasets: - dataset.wrap_intersection_points(sections, offset) - mock_other.wrap_intersection_points.assert_called_once_with( - sections, offset - ) + dataset.intersection_points(sections, offset) + mock_other.intersection_points.assert_called_once_with(sections, offset) def test_contained_by_section(self) -> None: mocked_datasets = self.get_mocked_datasets([], []) From 8835027fa5b5825fe85ff9dad5cec979378a231d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 17:39:58 +0200 Subject: [PATCH 089/128] Extract constants --- .../polars_geometry_store.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 0d73cc8c4..890668084 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -60,6 +60,8 @@ from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSet MAGNITUDE = "magnitude" +CUM_SUM = "cum_sum" +ORDER = "order" SEGMENT_LENGTH_X = "segment_length_x" SEGMENT_LENGTH_Y = "segment_length_y" @@ -1367,24 +1369,24 @@ def track_ids_after_cut(self, section: Section) -> pl.DataFrame: .select([ROW_ID, TRACK_ID, INTERSECTS]) ) - COLUMN_ORDER = [ROW_ID, TRACK_ID, INTERSECTS, "cum_sum", "order"] + COLUMN_ORDER = [ROW_ID, TRACK_ID, INTERSECTS, CUM_SUM, ORDER] results = ( result.with_columns( - pl.col(INTERSECTS).cum_sum().over(TRACK_ID).alias("cum_sum") + pl.col(INTERSECTS).cum_sum().over(TRACK_ID).alias(CUM_SUM) ) - .with_columns(pl.lit(1).alias("order")) + .with_columns(pl.lit(1).alias(ORDER)) .select(COLUMN_ORDER) ) temp = ( results.group_by(TRACK_ID) .first() - .with_columns(pl.lit(0).alias("order")) - .with_columns(pl.lit(0, pl.UInt32).alias("cum_sum")) + .with_columns(pl.lit(0).alias(ORDER)) + .with_columns(pl.lit(0, pl.UInt32).alias(CUM_SUM)) .with_columns(pl.col(ROW_ID) - 1) .select(COLUMN_ORDER) ) temp_r = pl.concat([results, temp]) - results = temp_r.sort(by=[TRACK_ID, ROW_ID, "order"]) + results = temp_r.sort(by=[TRACK_ID, ROW_ID, ORDER]) return results.with_columns( - pl.col(TRACK_ID) + "_" + pl.col("cum_sum").cast(pl.Utf8).alias(TRACK_ID) + pl.col(TRACK_ID) + "_" + pl.col(CUM_SUM).cast(pl.Utf8).alias(TRACK_ID) ).select([ROW_ID, TRACK_ID]) From 334b222b073b48e6f7049d1bee1f32a03b215bae Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 17:51:00 +0200 Subject: [PATCH 090/128] Fix version in precommit --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1312cbbb0..899edb7ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: - av==13.0.0 - black==24.8.0 - customtkinter==5.2.2 - - fastapi==0.115.5 + - fastapi==0.116.2 - hatch-requirements-txt==0.4.1 - ijson==3.3.0 - interrogate==1.7.0 @@ -55,7 +55,7 @@ repos: - matplotlib==3.10.0 - more-itertools==10.5.0 - mypy==1.16.1 - - nicegui==2.11.0 + - nicegui==2.24.1 - numpy==2.1.1 - pandas-stubs==2.2.3.241126 - pandas==2.2.3 @@ -67,7 +67,7 @@ repos: - pytest-cov==6.0.0 - pytest-lazy-fixtures==1.1.2 - pytest==8.3.4 - - starlette==0.41.2 + - starlette==0.48.0 - twine==6.1.0 - types-PyYAML - types-flake8 From 0d96b571ec03d15eff332bae4fb8a38604a35357 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 17:51:48 +0200 Subject: [PATCH 091/128] Fix version in precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 899edb7ec..4534e0d60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: - interrogate==1.7.0 - isort==5.13.2 - matplotlib==3.10.0 - - more-itertools==10.5.0 + - more-itertools==10.7.0 - mypy==1.16.1 - nicegui==2.24.1 - numpy==2.1.1 From b0432d479246756c5b9c2bbaebb7a2475354003c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 24 Sep 2025 17:58:07 +0200 Subject: [PATCH 092/128] Clean up benchmark --- tests/benchmark_otanalytics.py | 187 ++------------------------------- 1 file changed, 9 insertions(+), 178 deletions(-) diff --git a/tests/benchmark_otanalytics.py b/tests/benchmark_otanalytics.py index 104fe4be2..a43ce27d9 100644 --- a/tests/benchmark_otanalytics.py +++ b/tests/benchmark_otanalytics.py @@ -1,5 +1,4 @@ from datetime import datetime, timezone -from functools import cached_property from pathlib import Path from typing import Callable, Iterable @@ -51,46 +50,20 @@ from OTAnalytics.domain.track_repository import TrackRepository from OTAnalytics.domain.types import EventType from OTAnalytics.plugin_cli.cli_application import OtAnalyticsCliApplicationStarter -from OTAnalytics.plugin_datastore.filter_pandas_track_store import ( - FilterByClassPandasTrackDataset, -) -from OTAnalytics.plugin_datastore.polars_track_store import ( - POLARS_TRACK_GEOMETRY_FACTORY, - PolarsTrackDataset, -) -from OTAnalytics.plugin_datastore.python_track_store import ( - ByMaxConfidence, - FilteredPythonTrackDataset, - PythonTrackDataset, -) +from OTAnalytics.plugin_datastore.polars_track_store import PolarsTrackDataset from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( PolarsTrackGeometryDataset, ) from OTAnalytics.plugin_datastore.track_geometry_store.shapely_store import ( ShapelyTrackGeometryDataset, ) -from OTAnalytics.plugin_datastore.track_store import ( - PandasByMaxConfidence, - PandasTrackDataset, -) from OTAnalytics.plugin_intersect.simple.cut_tracks_with_sections import ( SimpleCutTracksIntersectingSection, ) from OTAnalytics.plugin_parser.feathers_parser import FeathersParser -from OTAnalytics.plugin_parser.otvision_parser import ( - OtFlowParser, - OttrkParser, - PythonDetectionParser, -) -from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser +from OTAnalytics.plugin_parser.otvision_parser import OtFlowParser from tests.utils.builders.run_configuration import create_run_config -PYTHON = "PYTHON" -PANDAS = "PANDAS" -PURE_PANDAS = "PURE_PANDAS" -POLARS = "POLARS" -CURRENT_DATASET_TYPE = POLARS - EXCLUDE_FILTER = [ OtcClasses.PEDESTRIAN, OtcClasses.BICYCLIST, @@ -116,22 +89,6 @@ def _fill_track_repository( return DetectionMetadata(frozenset(detection_classes)) -class BenchmarkOtAnalyticsStarter(OtAnalyticsCliApplicationStarter): - def __init__( - self, - run_config: RunConfiguration, - dataset_type: str, - ) -> None: - super().__init__(run_config) - self._dataset_type = dataset_type - - @cached_property - def track_geometry_factory(self) -> POLARS_TRACK_GEOMETRY_FACTORY: - if self._dataset_type == POLARS: - return PolarsTrackGeometryDataset.from_track_dataset - raise ValueError(f"Unknown dataset type {self._dataset_type}") - - class UseCaseProvider: GEOMETRY_FACTORY: TRACK_GEOMETRY_FACTORY = ( ShapelyTrackGeometryDataset.from_track_dataset @@ -163,7 +120,6 @@ def __init__( otflow_file: Path, ottrk_files: list[Path], save_dir: str, - dataset_type: str = CURRENT_DATASET_TYPE, ) -> None: self._otflow_file = otflow_file self._ottrk_files = ottrk_files @@ -171,9 +127,9 @@ def __init__( self._include_classes: frozenset[str] = frozenset() self._exclude_classes: frozenset[str] = frozenset() self._flow_parser = OtFlowParser() - self._starter = BenchmarkOtAnalyticsStarter(self.run_config, dataset_type) + self._starter = OtAnalyticsCliApplicationStarter(self.run_config) track_repository, detection_metadata = self.provide_track_repository( - self._ottrk_files, dataset_type + self._ottrk_files ) self._track_repository = track_repository self._detection_metadata = detection_metadata @@ -229,70 +185,14 @@ def add_filters( self._exclude_classes = frozenset(exclude_classes) def provide_track_repository( - self, track_files: list[Path], dataset_type: str + self, track_files: list[Path] ) -> tuple[TrackRepository, DetectionMetadata]: parser: TrackParser - if dataset_type == PYTHON: - repository = TrackRepository(self.provide_python_track_dataset()) - parser = OttrkParser(self.provide_python_detection_parser(repository)) - elif dataset_type == PANDAS: - repository = TrackRepository(self.provide_pandas_track_dataset()) - parser = OttrkParser(self.provide_pandas_detection_parser()) - elif dataset_type == PURE_PANDAS: - repository = TrackRepository(self.provide_pure_pandas_track_dataset()) - parser = FeathersParser() - elif dataset_type == POLARS: - repository = TrackRepository(self.provide_polars_track_dataset()) - parser = FeathersParser(PolarsTrackGeometryDataset.from_track_dataset) - else: - raise ValueError(f"Unknown dataset type {dataset_type}") + repository = TrackRepository(self.provide_polars_track_dataset()) + parser = FeathersParser(PolarsTrackGeometryDataset.from_track_dataset) detection_metadata = _fill_track_repository(parser, repository, track_files) return repository, detection_metadata - def provide_pandas_track_dataset(self) -> TrackDataset: - return FilterByClassPandasTrackDataset( - PandasTrackDataset.from_list( - [], ShapelyTrackGeometryDataset.from_track_dataset - ), - self._include_classes, - self._exclude_classes, - ) - - def provide_pandas_detection_parser(self) -> PandasDetectionParser: - return PandasDetectionParser( - PandasByMaxConfidence(), ShapelyTrackGeometryDataset.from_track_dataset - ) - - def provide_python_track_dataset(self) -> TrackDataset: - return FilteredPythonTrackDataset( - PythonTrackDataset(ShapelyTrackGeometryDataset.from_track_dataset), - self._include_classes, - self._exclude_classes, - ) - - def provide_python_detection_parser( - self, track_repository: TrackRepository - ) -> PythonDetectionParser: - return PythonDetectionParser( - ByMaxConfidence(), - track_repository, - ShapelyTrackGeometryDataset.from_track_dataset, - ) - - def provide_pure_pandas_track_dataset(self) -> TrackDataset: - return FilterByClassPandasTrackDataset( - PandasTrackDataset.from_list( - [], PolarsTrackGeometryDataset.from_track_dataset - ), - self._include_classes, - self._exclude_classes, - ) - - def provide_pure_pandas_detection_parser(self) -> PandasDetectionParser: - return PandasDetectionParser( - PandasByMaxConfidence(), PolarsTrackGeometryDataset.from_track_dataset - ) - def provide_polars_track_dataset(self) -> TrackDataset: return PolarsTrackDataset.from_list( [], PolarsTrackGeometryDataset.from_track_dataset @@ -328,17 +228,8 @@ def counting_specification(self, save_dir: Path) -> CountingSpecificationDto: def run_cli(self) -> Callable[[], None]: return self._starter.start - def get_track_parser(self, dataset_type: str = CURRENT_DATASET_TYPE) -> TrackParser: - if dataset_type == PYTHON: - return OttrkParser( - self.provide_python_detection_parser(self._track_repository) - ) - elif dataset_type == PANDAS: - return OttrkParser(self.provide_pandas_detection_parser()) - elif dataset_type == PURE_PANDAS: - return FeathersParser() - else: - raise ValueError(f"Unknown dataset type {dataset_type}") + def get_track_parser(self) -> TrackParser: + return FeathersParser() def retrieve_cutting_sections(sections: Iterable[Section]) -> list[Section]: @@ -370,28 +261,6 @@ def track_files_2hours(test_data_dir: Path) -> list[Path]: ] -@pytest.fixture(scope="module") -def track_files_4hours(test_data_dir: Path) -> list[Path]: - return [ - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-00-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-15-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-30-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_08-45-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-00-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-15-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-30-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_09-45-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-00-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-15-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-30-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_10-45-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-00-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-15-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-30-00.ottrk"), - Path(test_data_dir / "OTCamera19_FR20_2023-05-24_11-45-00.ottrk"), - ] - - @pytest.fixture(scope="module") def otflow_file(test_data_dir: Path) -> Path: return test_data_dir / Path("OTCamera19_FR20_2023-05-24.otflow") @@ -410,25 +279,6 @@ def cutting_section() -> Section: ) -@pytest.fixture -def python_track_parser(python_track_repository: TrackRepository) -> TrackParser: - detection_parser = PythonDetectionParser( - ByMaxConfidence(), - python_track_repository, - ShapelyTrackGeometryDataset.from_track_dataset, - ) - return OttrkParser(detection_parser) - - -@pytest.fixture -def pandas_track_parser() -> TrackParser: - calculator = PandasByMaxConfidence() - detection_parser = PandasDetectionParser( - calculator, ShapelyTrackGeometryDataset.from_track_dataset - ) - return OttrkParser(detection_parser) - - @pytest.fixture def feathers_track_parser() -> TrackParser: return FeathersParser() @@ -448,13 +298,6 @@ def use_case_provider_2hours( return UseCaseProvider(otflow_file, track_files_2hours, str(test_data_tmp_dir)) -@pytest.fixture -def use_case_provider_4hours( - otflow_file: Path, track_files_4hours: list[Path], test_data_tmp_dir: Path -) -> UseCaseProvider: - return UseCaseProvider(otflow_file, track_files_4hours, str(test_data_tmp_dir)) - - @pytest.fixture def use_case_provider_15min_filtered( otflow_file: Path, track_file_15min: Path, test_data_tmp_dir: Path @@ -544,18 +387,6 @@ def test_2hours( warmup_rounds=self.WARMUP_ROUNDS, ) - def test_4hours( - self, benchmark: BenchmarkFixture, use_case_provider_4hours: UseCaseProvider - ) -> None: - use_case = use_case_provider_4hours.get_tracks_intersecting_sections() - benchmark.pedantic( - use_case, - args=(use_case_provider_4hours.sections,), - rounds=self.ROUNDS, - iterations=self.ITERATIONS, - warmup_rounds=self.WARMUP_ROUNDS, - ) - def test_15min_filtered( self, benchmark: BenchmarkFixture, From f8b7ecc0ccb64fc7379f3de357808b0318aad09c Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 25 Sep 2025 10:15:06 +0200 Subject: [PATCH 093/128] Implement missing method --- .../plugin_datastore/polars_track_store.py | 42 ++++-- .../test_polars_track_store.py | 127 ++++++++++++++++++ 2 files changed, 157 insertions(+), 12 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 3e8d6b836..551ab3ae6 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from collections import defaultdict from dataclasses import dataclass from datetime import datetime from math import ceil @@ -25,7 +26,7 @@ from OTAnalytics.domain import track from OTAnalytics.domain.geometry import RelativeOffsetCoordinate from OTAnalytics.domain.section import Section, SectionId -from OTAnalytics.domain.track import Detection, Track, TrackId, unpack +from OTAnalytics.domain.track import Detection, Track, TrackId, pack, unpack from OTAnalytics.domain.track_dataset.track_dataset import ( END_FRAME, END_OCCURRENCE, @@ -498,16 +499,7 @@ def intersection_points( def ids_inside(self, sections: list[Section]) -> TrackIdSet: result: TrackIdSet = PolarsTrackIdSet() for _section in sections: - section_geom = area_section_to_shapely(_section) - offset = _section.get_offset(EventType.SECTION_ENTER) - x_coords = ( - self._dataset[track.X] + offset.x * self._dataset[track.W] - ).to_numpy() - y_coords = ( - self._dataset[track.Y] + offset.y * self._dataset[track.H] - ).to_numpy() - - contains_masks = contains(section_geom, x_coords, y_coords) + contains_masks = self.__calculate_contains_by_section_mask(_section) result = result.union( PolarsTrackIdSet( self._dataset.filter(contains_masks) @@ -518,10 +510,36 @@ def ids_inside(self, sections: list[Section]) -> TrackIdSet: ) return result + def __calculate_contains_by_section_mask(self, section: Section) -> numpy.ndarray: + section_geom = area_section_to_shapely(section) + offset = section.get_offset(EventType.SECTION_ENTER) + x_coords = ( + self._dataset[track.X] + offset.x * self._dataset[track.W] + ).to_numpy() + y_coords = ( + self._dataset[track.Y] + offset.y * self._dataset[track.H] + ).to_numpy() + return contains(section_geom, x_coords, y_coords) + def contained_by_sections( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> dict[TrackId, list[tuple[SectionId, list[bool]]]]: - raise NotImplementedError + contains_result: dict[TrackId, list[tuple[SectionId, list[bool]]]] = ( + defaultdict(list) + ) + for _section in sections: + contains_masks = self.__calculate_contains_by_section_mask(_section) + if any(contains_masks): + temp = ( + self._dataset.with_columns( + pl.Series("contains_mask", contains_masks) + ) + .group_by(pl.col(track.TRACK_ID)) + .agg(pl.col("contains_mask").alias("grouped")) + ) + for track_id, contains_mask in temp.iter_rows(): + contains_result[pack(track_id)].append((_section.id, contains_mask)) + return contains_result def _add_to_geometry_dataset(self, dataset: "PolarsTrackDataset") -> dict: """Add to geometry dataset - placeholder implementation.""" diff --git a/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py index df0204187..3436b94c0 100644 --- a/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py +++ b/tests/OTAnalytics/plugin_datastore/test_polars_track_store.py @@ -1,8 +1,12 @@ +from dataclasses import dataclass from unittest.mock import Mock, call import polars as pl +import pytest from OTAnalytics.domain import track +from OTAnalytics.domain.geometry import Coordinate, RelativeOffsetCoordinate +from OTAnalytics.domain.section import LineSection, Section, SectionId from OTAnalytics.domain.track import Track, TrackId from OTAnalytics.plugin_datastore.polars_track_id_set import PolarsTrackIdSet from OTAnalytics.plugin_datastore.polars_track_store import ( @@ -14,10 +18,114 @@ _convert_tracks, create_empty_dataframe, ) +from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( + PolarsTrackGeometryDataset, +) from tests.utils.assertions import assert_equal_track_properties from tests.utils.builders.track_builder import TrackBuilder +def create_line_section( + section_id: str, coordinates: list[tuple[float, float]] +) -> Section: + section = Mock(spec=LineSection) + section.get_coordinates.return_value = [ + Coordinate(coord[0], coord[1]) for coord in coordinates + ] + section.get_offset.return_value = RelativeOffsetCoordinate(0.0, 0.0) + section.id = SectionId(section_id) + return section + + +@dataclass +class ContainedBySectionTestCase: + tracks: list[Track] + sections: list[Section] + expected_result: dict[TrackId, list[tuple[SectionId, list[bool]]]] + + +@pytest.fixture +def contained_by_section_test_case( + straight_track: Track, complex_track: Track, closed_track: Track +) -> ContainedBySectionTestCase: + # Straight track starts outside section + first_section = create_line_section( + "1", [(1.5, 0.5), (1.5, 1.5), (2.5, 1.5), (2.5, 0.5), (1.5, 0.5)] + ) + # Straight track starts inside section + second_section = create_line_section( + "2", [(0.5, 0.5), (0.5, 1.5), (1.5, 1.5), (1.5, 0.5), (0.5, 0.5)] + ) + # Straight track is inside section + third_section = create_line_section( + "3", [(0.0, 0.0), (0.0, 2.0), (4.0, 2.0), (4.0, 0.0), (0.0, 0.0)] + ) + # Straight track starts outside stays inside section + fourth_section = create_line_section( + "4", [(1.5, 0.5), (1.5, 1.5), (4.0, 1.5), (4.0, 0.5), (1.5, 0.5)] + ) + # Complex track starts outside section with multiple intersections + fifth_section = create_line_section( + "5", + [(1.5, 0.5), (1.5, 2.5), (2.5, 2.5), (2.5, 0.5), (1.5, 0.5)], + ) + # Complex track starts inside section with multiple intersections + sixth_section = create_line_section( + "6", [(0.5, 0.5), (0.5, 2.5), (1.5, 2.5), (1.5, 0.5), (0.5, 0.5)] + ) + # Closed track + seventh_section = create_line_section( + "7", [(1.5, 0.5), (1.5, 2.0), (2.5, 2.0), (2.5, 0.5), (1.5, 0.5)] + ) + # Not contained track + eighth_section = create_line_section( + "not-contained", [(3.0, 1.0), (3.0, 2.0), (4.0, 2.0), (4.0, 1.0), (3.0, 1.0)] + ) + expected = { + straight_track.id: [ + (first_section.id, [False, True, False]), + (second_section.id, [True, False, False]), + (third_section.id, [True, True, True]), + (fourth_section.id, [False, True, True]), + (fifth_section.id, [False, True, False]), + (sixth_section.id, [True, False, False]), + (seventh_section.id, [False, True, False]), + ], + complex_track.id: [ + (first_section.id, [False, True, False, False, False, False]), + (second_section.id, [True, False, False, False, False, False]), + (third_section.id, [True, True, True, True, False, False]), + (fourth_section.id, [False, True, False, False, False, False]), + (fifth_section.id, [False, True, True, False, False, True]), + (sixth_section.id, [True, False, False, True, True, False]), + (seventh_section.id, [False, True, True, False, False, False]), + ], + closed_track.id: [ + (first_section.id, [False, True, False, False, False]), + (second_section.id, [True, False, False, False, True]), + (third_section.id, [True, True, False, False, True]), + (fourth_section.id, [False, True, False, False, False]), + (fifth_section.id, [False, True, True, False, False]), + (sixth_section.id, [True, False, False, True, True]), + (seventh_section.id, [False, True, False, False, False]), + ], + } + return ContainedBySectionTestCase( + [straight_track, complex_track, closed_track], + [ + first_section, + second_section, + third_section, + fourth_section, + fifth_section, + sixth_section, + seventh_section, + eighth_section, + ], + expected, + ) + + class TestPolarsDetection: def test_properties(self) -> None: track_id = "track_1" @@ -380,6 +488,25 @@ def test_create_test_flyweight_with_single_detection( assert result.id == track.id assert len(result.detections) == 1 + def test_contained_by_sections( + self, + contained_by_section_test_case: ContainedBySectionTestCase, + ) -> None: + unused_offset = RelativeOffsetCoordinate(0, 0) + track_dataset = PolarsTrackDataset.from_list( + contained_by_section_test_case.tracks, + PolarsTrackGeometryDataset.from_track_dataset, + ) + result = track_dataset.contained_by_sections( + contained_by_section_test_case.sections, unused_offset + ) + expected_result = contained_by_section_test_case.expected_result + for track_id in expected_result: + result_data = result[track_id] + expected_data = expected_result[track_id] + assert result_data == expected_data + assert result == expected_result + def test_convert_tracks() -> None: builder = TrackBuilder().add_track_id("1") From 5d871dde40241edc9125b6f5debbededac0f7044 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Thu, 25 Sep 2025 10:19:33 +0200 Subject: [PATCH 094/128] Add documentation --- .../plugin_datastore/polars_track_store.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 551ab3ae6..44239a2b7 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -275,6 +275,23 @@ def drop_row_id(df: pl.DataFrame) -> pl.DataFrame: class PolarsTrackDataset(TrackDataset, PolarsDataFrameProvider): + """High-performance TrackDataset implementation using Polars for vectorized + operations. + + This implementation leverages Polars DataFrames to provide significant performance + improvements for large-scale track data processing through vectorized operations. + + Key features: + - Vectorized intersection calculations + - Efficient memory usage through lazy evaluation + - Optimized filtering and aggregation operations + + Note: + This implementation may have different performance characteristics + compared to other TrackDataset implementations, particularly for + small datasets where setup overhead may outweigh benefits. + """ + @property def track_ids(self) -> TrackIdSet: if self._dataset.is_empty(): @@ -466,6 +483,22 @@ def __len__(self) -> int: def intersecting_tracks( self, sections: list[Section], offset: RelativeOffsetCoordinate ) -> TrackIdSet: + """Return a set of tracks intersecting a set of sections. + + This implementation uses Polars-based vectorized geometry operations for + improved performance on large track datasets. + + Args: + sections (list[Section]): the list of sections to intersect. + offset (RelativeOffsetCoordinate): the offset to be applied to the tracks. + + Returns: + TrackIdSet: the track ids intersecting the given sections. + + Performance: + O(n*m) where n is number of tracks and m is number of sections, + but with vectorized operations for significant speedup on large datasets. + """ geometry_dataset = self._get_geometry_dataset_for(offset) return geometry_dataset.intersecting_tracks(sections) From f9ce9ee6f7f02bf2328f0f72f8add7e092c8a993 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:33:22 +0200 Subject: [PATCH 095/128] Replace `print` statements with `logger` for consistent logging and update metadata filename generation with a constant suffix. --- .../convert_ottrk_to_feathers.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py index 33c99207f..48df20419 100644 --- a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py +++ b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py @@ -2,6 +2,7 @@ from typing import Any, Dict from OTAnalytics.application.datastore import TrackParseResult +from OTAnalytics.application.logger import logger from OTAnalytics.domain.track_dataset.track_dataset import TRACK_GEOMETRY_FACTORY from OTAnalytics.plugin_datastore.python_track_store import PythonTrackDataset from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( @@ -75,13 +76,13 @@ def convert_ottrk_to_feather(input_file: Path) -> None: feather_file = output_dir / f"{output_stem}.feather" metadata_file = output_dir / f"{output_stem}_metadata.json" - print(f"Reading ottrk file: {input_file}") + logger().info(f"Reading ottrk file: {input_file}") # Parse the ottrk file parser = create_ottrk_parser() parse_result = parser.parse(input_file) - print(f"Parsed {len(parse_result.tracks)} tracks") + logger().info(f"Parsed {len(parse_result.tracks)} tracks") if isinstance(parse_result.tracks, PandasTrackDataset): pandas_dataset = parse_result.tracks @@ -90,18 +91,18 @@ def convert_ottrk_to_feather(input_file: Path) -> None: # Get the pandas DataFrame df = pandas_dataset.get_data() - print(f"DataFrame shape: {df.shape}") + logger().info(f"DataFrame shape: {df.shape}") # Save DataFrame to feather format - print(f"Saving DataFrame to: {feather_file}") + logger().info(f"Saving DataFrame to: {feather_file}") df.to_feather(feather_file) # Create and save metadata metadata = create_metadata_dict(parse_result) - print(f"Saving metadata to: {metadata_file}") + logger().info(f"Saving metadata to: {metadata_file}") write_json(metadata, metadata_file) - print("Conversion completed successfully!") - print("Output files:") - print(f" - Data: {feather_file}") - print(f" - Metadata: {metadata_file}") + logger().info("Conversion completed successfully!") + logger().info("Output files:") + logger().info(f" - Data: {feather_file}") + logger().info(f" - Metadata: {metadata_file}") From 227678e093889b408fdaad68ce10939244500d22 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 2 Oct 2025 16:33:46 +0200 Subject: [PATCH 096/128] Refactor: Introduce constants for file extensions and update references in feather and metadata file handling. --- .../plugin_parser/convert_ottrk_to_feathers.py | 7 +++++-- OTAnalytics/plugin_parser/feathers_parser.py | 16 ++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py index 48df20419..268f6ca8c 100644 --- a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py +++ b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py @@ -19,6 +19,9 @@ ) from OTAnalytics.plugin_parser.pandas_parser import PandasDetectionParser +METADATA_SUFFIX = "_metadata.json" +FEATHER_FILETYPE = ".feather" + def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: """Create a track geometry factory for PandasTrackDataset.""" @@ -73,8 +76,8 @@ def convert_ottrk_to_feather(input_file: Path) -> None: # Create output file paths using the same stem as input output_stem = input_file.stem output_dir = input_file.parent - feather_file = output_dir / f"{output_stem}.feather" - metadata_file = output_dir / f"{output_stem}_metadata.json" + feather_file = output_dir / f"{output_stem}{FEATHER_FILETYPE}" + metadata_file = output_dir / f"{output_stem}{METADATA_SUFFIX}" logger().info(f"Reading ottrk file: {input_file}") diff --git a/OTAnalytics/plugin_parser/feathers_parser.py b/OTAnalytics/plugin_parser/feathers_parser.py index 128452480..a39204ac6 100644 --- a/OTAnalytics/plugin_parser/feathers_parser.py +++ b/OTAnalytics/plugin_parser/feathers_parser.py @@ -27,7 +27,11 @@ from OTAnalytics.plugin_datastore.track_geometry_store.polars_geometry_store import ( PolarsTrackGeometryDataset, ) -from OTAnalytics.plugin_parser.convert_ottrk_to_feathers import convert_ottrk_to_feather +from OTAnalytics.plugin_parser.convert_ottrk_to_feathers import ( + FEATHER_FILETYPE, + METADATA_SUFFIX, + convert_ottrk_to_feather, +) from OTAnalytics.plugin_parser.json_parser import parse_json @@ -47,14 +51,14 @@ def use_feathers_files(files: list[Path]) -> list[Path]: def use_feather_file(file: Path) -> Path: - if not file.suffix.lower() == ".feather": + if not file.suffix.lower() == FEATHER_FILETYPE: if file.suffix.lower() == ".ottrk": - if not file.with_suffix(".feather").exists(): + if not file.with_suffix(FEATHER_FILETYPE).exists(): convert_ottrk_to_feather(file) - return file.with_suffix(".feather") + return file.with_suffix(FEATHER_FILETYPE) else: raise ValueError( - f"Input file must have .feather or .ottrk extension: {file}" + f"Input file must have {FEATHER_FILETYPE} or .ottrk extension: {file}" ) return file @@ -108,7 +112,7 @@ def parse_files(self, files: list[Path]) -> TracksParseResult: if not file.exists(): raise FileNotFoundError(f"Feather file not found: {file}") # Construct metadata file path - metadata_file = file.parent / f"{file.stem}_metadata.json" + metadata_file = file.parent / f"{file.stem}{METADATA_SUFFIX}" if not metadata_file.exists(): raise FileNotFoundError(f"Metadata file not found: {metadata_file}") From 77fd2aa7e0a94bf5c759051e3ded6d744e349ff1 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 2 Oct 2025 16:34:20 +0200 Subject: [PATCH 097/128] Add TODO comments to check and potentially remove unimplemented methods --- .../track_geometry_store/polars_geometry_store.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 890668084..d4792a1e3 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -613,15 +613,19 @@ def __len__(self) -> int: return len(self._events) def __add__(self, other: "EventDataset") -> "EventDataset": + # TODO: Check to remove raise NotImplementedError def __eq__(self, other: object) -> bool: + # TODO: Check to remove raise NotImplementedError def extend(self, other: "EventDataset") -> None: + # TODO: Check to remove raise NotImplementedError def append(self, event: Event) -> None: + # TODO: Check to remove raise NotImplementedError def is_empty(self) -> bool: From f1ef2bc71d17df63f0a98dac5aad8a0e555954cb Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 2 Oct 2025 18:59:29 +0200 Subject: [PATCH 098/128] Remove duplicate import in `test_inside_cutting_section` test. --- .../application/use_cases/test_inside_cutting_section.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py index cc3151600..d9bea06da 100644 --- a/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py +++ b/tests/OTAnalytics/application/use_cases/test_inside_cutting_section.py @@ -56,8 +56,6 @@ def cached_track_ids_inside_cutting_sections() -> CachedTrackIdsInsideCuttingSec cached_track_ids_inside_cutting_sections = CachedTrackIdsInsideCuttingSections( get_all_tracks, get_cutting_sections ) - from OTAnalytics.plugin_datastore.python_track_store import PythonTrackIdSet - cached_track_ids_inside_cutting_sections._cached_ids = PythonTrackIdSet( { TrackId("1"), From 3026172e36c25ba5de8953d2aecaaa0ca73867b5 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 2 Oct 2025 19:45:35 +0200 Subject: [PATCH 099/128] Add assertions for `update_detection_classes` calls in `TestLoadTrackFile` --- .../use_cases/test_load_track_files.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/OTAnalytics/application/use_cases/test_load_track_files.py b/tests/OTAnalytics/application/use_cases/test_load_track_files.py index 76fc70628..9f84a86ea 100644 --- a/tests/OTAnalytics/application/use_cases/test_load_track_files.py +++ b/tests/OTAnalytics/application/use_cases/test_load_track_files.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from pathlib import Path -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock, Mock, call from OTAnalytics.application.datastore import TracksParseResult from OTAnalytics.application.use_cases.load_track_files import LoadTrackFiles @@ -13,12 +13,13 @@ class TestLoadTrackFile: def test_load_multiple_files(self) -> None: + classes = {"class1", "class2"} given = setup( track_ids=[TrackId("1"), TrackId("2")], video_files=[Path("video1.mp4"), Path("video2.mp4")], track_files=[some_file, other_file], existing_track_files=[], - classes={"class1", "class2"}, + classes=classes, ) target = create_target(given) @@ -32,6 +33,10 @@ def test_load_multiple_files(self) -> None: given.track_file_repository.add_all.assert_called_once_with( [some_file, other_file] ) + assert given.tracks_metadata.update_detection_classes.call_args_list == [ + call(classes), + call(classes), + ] def test_load_existing_track_file(self) -> None: """ @@ -55,6 +60,7 @@ def test_load_existing_track_file(self) -> None: given.video_repository.add_all.assert_not_called() given.track_repository.add_all.assert_not_called() given.track_file_repository.add_all.assert_not_called() + given.tracks_metadata.update_detection_classes.assert_not_called() def test_load_multiple_with_existing_track_file(self) -> None: """ @@ -62,12 +68,13 @@ def test_load_multiple_with_existing_track_file(self) -> None: @bug by randy-seng """ # noqa + classes = {"class1"} given = setup( track_ids=[TrackId("1")], video_files=[Path("video1.mp4")], - track_files=[some_file, other_file], + track_files=[other_file], existing_track_files=[some_file], - classes={"class1"}, + classes=classes, ) target = create_target(given) @@ -80,6 +87,7 @@ def test_load_multiple_with_existing_track_file(self) -> None: given.parse_result.tracks ) given.track_file_repository.add_all.assert_called_once_with([other_file]) + given.tracks_metadata.update_detection_classes.assert_called_once_with(classes) def test_load_empty_files_list(self) -> None: given = setup( @@ -98,6 +106,7 @@ def test_load_empty_files_list(self) -> None: given.video_repository.add_all.assert_not_called() given.track_repository.add_all.assert_not_called() given.track_file_repository.add_all.assert_not_called() + given.tracks_metadata.update_detection_classes.assert_not_called() def test_load_with_videos_metadata_update(self) -> None: given = setup( From b236979c805fb10e8305ce00205a3e2515beb379 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 11:57:08 +0200 Subject: [PATCH 100/128] Reintroduce removed types for mypy. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4534e0d60..17cbe8dcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,6 +57,7 @@ repos: - mypy==1.16.1 - nicegui==2.24.1 - numpy==2.1.1 + - opencv-python-headless==4.10.0.84 - pandas-stubs==2.2.3.241126 - pandas==2.2.3 - polars==1.32.3 @@ -73,6 +74,7 @@ repos: - types-flake8 - types-openpyxl - types-pillow + - types-pywin32 - types-requests - types-seaborn - types-selenium From be93ae7e9a1fc104e4e6259896afd90e91ba3d0d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:01:59 +0200 Subject: [PATCH 101/128] Sort ids --- OTAnalytics/plugin_datastore/polars_track_id_set.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_id_set.py b/OTAnalytics/plugin_datastore/polars_track_id_set.py index e4accd17e..204be6e7b 100644 --- a/OTAnalytics/plugin_datastore/polars_track_id_set.py +++ b/OTAnalytics/plugin_datastore/polars_track_id_set.py @@ -40,7 +40,9 @@ def __len__(self) -> int: def intersection(self, other: "TrackIdSet") -> "TrackIdSet": if isinstance(other, PolarsTrackIdSet): # Efficient intersection using Polars operations - intersected_series = self._series.filter(self._series.is_in(other._series)) + intersected_series = self._series.filter( + self._series.is_in(other._series) + ).sort() return PolarsTrackIdSet(intersected_series) else: # Convert other to set of strings for intersection @@ -65,14 +67,16 @@ def union(self, other: "TrackIdSet") -> "TrackIdSet": def difference(self, other: "TrackIdSet") -> "TrackIdSet": if isinstance(other, PolarsTrackIdSet): # Efficient difference using Polars operations - difference_series = self._series.filter(~self._series.is_in(other._series)) + difference_series = self._series.filter( + ~self._series.is_in(other._series) + ).sort() return PolarsTrackIdSet(difference_series) else: # Convert other to set of strings for difference other_strings = {unpack(track_id) for track_id in other} difference_series = self._series.filter( ~self._series.is_in(list(other_strings)) - ) + ).sort() return PolarsTrackIdSet(difference_series) From c2c79551e536450786f3174388eeab363f441370 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:03:47 +0200 Subject: [PATCH 102/128] Rename method --- OTAnalytics/plugin_datastore/polars_track_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 44239a2b7..199b42d4b 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -455,7 +455,7 @@ def remove(self, track_id: TrackId) -> "PolarsTrackDataset": ) def remove_multiple(self, track_ids: TrackIdSet) -> "PolarsTrackDataset": - track_id_strings = self.__to_polars_series(track_ids) + track_id_strings = self.__to_raw_ids(track_ids) filtered_data = self._dataset.filter( ~pl.col(LEVEL_TRACK_ID).is_in(track_id_strings) ) @@ -466,7 +466,7 @@ def remove_multiple(self, track_ids: TrackIdSet) -> "PolarsTrackDataset": self.calculator, ) - def __to_polars_series(self, track_ids: TrackIdSet) -> pl.Series | list: + def __to_raw_ids(self, track_ids: TrackIdSet) -> pl.Series | list: if isinstance(track_ids, PolarsTrackIdSet): return track_ids._series return [unpack(track_id) for track_id in track_ids] From e9b56a6f911cacf1a4001db126d3f8f9015a881d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:05:15 +0200 Subject: [PATCH 103/128] Remove unnecessary local import --- OTAnalytics/plugin_datastore/polars_track_store.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 199b42d4b..95ae37a09 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -685,8 +685,6 @@ def cut_with_section( ) -> tuple["PolarsTrackDataset", TrackIdSet]: if len(self) == 0: logger().info("No tracks to cut") - from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet - return self, EmptyTrackIdSet() geometry_dataset = self._get_geometry_dataset_for(offset) From d8ba935dbe46d0f672f549dc48345729f725f9b2 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:08:31 +0200 Subject: [PATCH 104/128] Return old track ids when cutting Was missing in implementation until now. --- OTAnalytics/plugin_datastore/polars_track_store.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 95ae37a09..059b26841 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -689,13 +689,16 @@ def cut_with_section( geometry_dataset = self._get_geometry_dataset_for(offset) new_track_ids = geometry_dataset.track_ids_after_cut(section) + original_track_ids = PolarsTrackIdSet( + self._dataset.get_column(track.TRACK_ID).unique() + ) old_track_ids = self._dataset.rename({track.TRACK_ID: "old_track_id"}) result = old_track_ids.join(new_track_ids, on=ROW_ID, how="left") result = result.drop("old_track_id") return ( PolarsTrackDataset.from_dataframe(result, self.track_geometry_factory), - PolarsTrackIdSet(), + original_track_ids, ) def filter_by_min_detection_length(self, length: int) -> TrackDataset: From a08848938121e38c12700fcbbb3ca64d9f4a0cfc Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:11:37 +0200 Subject: [PATCH 105/128] Use unpack where possible --- OTAnalytics/plugin_datastore/polars_track_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 059b26841..b6c4a2af3 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -728,7 +728,7 @@ def get_max_confidences_for(self, track_ids: TrackIdSet) -> dict[str, float]: return {} try: - track_id_strings = [track_id.id for track_id in track_ids] + track_id_strings = [unpack(track_id) for track_id in track_ids] result = ( self._dataset.filter(pl.col(LEVEL_TRACK_ID).is_in(track_id_strings)) .group_by(LEVEL_TRACK_ID) @@ -762,7 +762,7 @@ def revert_cuts_for( return self, EmptyTrackIdSet(), EmptyTrackIdSet() # Revert track IDs back to original IDs - ids_to_revert_strings = [track_id.id for track_id in ids_to_revert] + ids_to_revert_strings = [unpack(track_id) for track_id in ids_to_revert] result = self._dataset.with_columns( pl.when(pl.col(LEVEL_TRACK_ID).is_in(ids_to_revert_strings)) .then(pl.col(track.ORIGINAL_TRACK_ID)) From 58e9a83c55eadcee6cff3c5cd8c4e7edf3b13ce3 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:12:26 +0200 Subject: [PATCH 106/128] Use unpack where possible --- OTAnalytics/plugin_datastore/polars_track_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index b6c4a2af3..7a31ac6c8 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -787,7 +787,7 @@ def _get_existing_track_ids(self, track_ids: TrackIdSet) -> TrackIdSet: return EmptyTrackIdSet() - converted_ids = [track_id.id for track_id in track_ids] + converted_ids = [unpack(track_id) for track_id in track_ids] existing_ids = ( self._dataset.filter(pl.col(track.ORIGINAL_TRACK_ID).is_in(converted_ids)) .get_column(LEVEL_TRACK_ID) From 6bba8a655414c7dbca202efcbc9d369fe21f5172 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:13:46 +0200 Subject: [PATCH 107/128] Remove unnecessary conversion --- OTAnalytics/plugin_datastore/polars_track_store.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 7a31ac6c8..4fafe54c7 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -792,9 +792,8 @@ def _get_existing_track_ids(self, track_ids: TrackIdSet) -> TrackIdSet: self._dataset.filter(pl.col(track.ORIGINAL_TRACK_ID).is_in(converted_ids)) .get_column(LEVEL_TRACK_ID) .unique() - .to_list() ) - return PolarsTrackIdSet([TrackId(track_id) for track_id in existing_ids]) + return PolarsTrackIdSet(existing_ids) def split(self, batches: int) -> Sequence["PolarsTrackDataset"]: dataset_size = len(self) From a9ea720351dc17bb0497524d2a94ccd8f4c228da Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:29:13 +0200 Subject: [PATCH 108/128] Change calculation of intersection parameters to follow the normally taught version of cramer's rule. --- .../track_geometry_store/polars_geometry_store.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index d4792a1e3..b469207b7 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -236,8 +236,8 @@ def calculate_intersection_parameters( [ # Calculate denominator for intersection formula ( - pl.col("line_dx") * pl.col("seg_dy") - - pl.col("line_dy") * pl.col("seg_dx") + pl.col("line_dy") * pl.col("seg_dx") + - pl.col("line_dx") * pl.col("seg_dy") ).alias(DENOMINATOR), ] ) @@ -255,8 +255,8 @@ def calculate_intersection_parameters( pl.when(pl.col(NON_PARALLEL)) .then( ( - (pl.col("seg_x1") - line_x1) * pl.col("seg_dy") - - (pl.col("seg_y1") - line_y1) * pl.col("seg_dx") + pl.col("seg_dx") * (pl.col("seg_y1") - line_y1) + - pl.col("seg_dy") * (pl.col("seg_x1") - line_x1) ) / pl.col(DENOMINATOR) ) @@ -265,8 +265,8 @@ def calculate_intersection_parameters( pl.when(pl.col(NON_PARALLEL)) .then( ( - (pl.col("seg_x1") - line_x1) * pl.col("line_dy") - - (pl.col("seg_y1") - line_y1) * pl.col("line_dx") + pl.col("line_dx") * (pl.col("seg_y1") - line_y1) + - pl.col("line_dy") * (pl.col("seg_x1") - line_x1) ) / pl.col(DENOMINATOR) ) From ddfc1067007dc30e3233f71add0fb91794c70b88 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:45:46 +0200 Subject: [PATCH 109/128] Fix assertions in regression tests --- tests/utils/assertions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/assertions.py b/tests/utils/assertions.py index a37ecc609..705177e58 100644 --- a/tests/utils/assertions.py +++ b/tests/utils/assertions.py @@ -68,11 +68,11 @@ def assert_two_files_equal_sorted( with open(actual_counts_file, mode="r") as actual: act_lines = sorted(actual.readlines()) - act_lines = sorted([line.split(",", 1)[1] for line in actual.readlines()]) + act_lines = sorted([line.split(",", 1)[1] for line in act_lines]) with open(expected_counts_file, mode="r") as expected: exp_lines = sorted(expected.readlines()) - exp_lines = sorted([line.split(",", 1)[1] for line in expected.readlines()]) + exp_lines = sorted([line.split(",", 1)[1] for line in exp_lines]) assert set(act_lines) == set(exp_lines), ( f"Sets are not equal [act len: {len(act_lines)}, " From ab9a6b1f1a0d80395efaad28e00cf30183d4d3b9 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 6 Oct 2025 12:46:54 +0200 Subject: [PATCH 110/128] Simplify file loading in assertions --- tests/utils/assertions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/utils/assertions.py b/tests/utils/assertions.py index 705177e58..8e5c52b89 100644 --- a/tests/utils/assertions.py +++ b/tests/utils/assertions.py @@ -67,12 +67,10 @@ def assert_two_files_equal_sorted( ) -> None: with open(actual_counts_file, mode="r") as actual: - act_lines = sorted(actual.readlines()) - act_lines = sorted([line.split(",", 1)[1] for line in act_lines]) + act_lines = sorted([line.split(",", 1)[1] for line in actual.readlines()]) with open(expected_counts_file, mode="r") as expected: - exp_lines = sorted(expected.readlines()) - exp_lines = sorted([line.split(",", 1)[1] for line in exp_lines]) + exp_lines = sorted([line.split(",", 1)[1] for line in expected.readlines()]) assert set(act_lines) == set(exp_lines), ( f"Sets are not equal [act len: {len(act_lines)}, " From 526e5ff4f89a47e7bb15fa91c4694d20ae0c6db8 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 8 Oct 2025 17:01:40 +0200 Subject: [PATCH 111/128] Fix intersection calculation --- .../track_geometry_store/polars_geometry_store.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index b469207b7..d4792a1e3 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -236,8 +236,8 @@ def calculate_intersection_parameters( [ # Calculate denominator for intersection formula ( - pl.col("line_dy") * pl.col("seg_dx") - - pl.col("line_dx") * pl.col("seg_dy") + pl.col("line_dx") * pl.col("seg_dy") + - pl.col("line_dy") * pl.col("seg_dx") ).alias(DENOMINATOR), ] ) @@ -255,8 +255,8 @@ def calculate_intersection_parameters( pl.when(pl.col(NON_PARALLEL)) .then( ( - pl.col("seg_dx") * (pl.col("seg_y1") - line_y1) - - pl.col("seg_dy") * (pl.col("seg_x1") - line_x1) + (pl.col("seg_x1") - line_x1) * pl.col("seg_dy") + - (pl.col("seg_y1") - line_y1) * pl.col("seg_dx") ) / pl.col(DENOMINATOR) ) @@ -265,8 +265,8 @@ def calculate_intersection_parameters( pl.when(pl.col(NON_PARALLEL)) .then( ( - pl.col("line_dx") * (pl.col("seg_y1") - line_y1) - - pl.col("line_dy") * (pl.col("seg_x1") - line_x1) + (pl.col("seg_x1") - line_x1) * pl.col("line_dy") + - (pl.col("seg_y1") - line_y1) * pl.col("line_dx") ) / pl.col(DENOMINATOR) ) From c4196dd1866d0bcf9714bce8e50a0396064d5322 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 2 Oct 2025 20:23:25 +0200 Subject: [PATCH 112/128] Refactor: Introduce constants for metadata keys and update references in `convert_ottrk_to_feathers.py` and `feathers_parser.py`. --- .../plugin_parser/convert_ottrk_to_feathers.py | 11 ++++++++--- OTAnalytics/plugin_parser/feathers_parser.py | 11 +++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py index 268f6ca8c..f7db205b5 100644 --- a/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py +++ b/OTAnalytics/plugin_parser/convert_ottrk_to_feathers.py @@ -21,6 +21,9 @@ METADATA_SUFFIX = "_metadata.json" FEATHER_FILETYPE = ".feather" +KEY_DETECTION_METADATA = "detection_metadata" +KEY_DETECTION_CLASSES = "detection_classes" +KEY_VIDEO_METADATA = "video_metadata" def create_track_geometry_factory() -> TRACK_GEOMETRY_FACTORY: @@ -52,10 +55,12 @@ def convert_to_pandas_dataset( def create_metadata_dict(parse_result: TrackParseResult) -> Dict[str, Any]: """Create a metadata dictionary from TrackParseResult.""" metadata = { - "detection_metadata": { - "detection_classes": list(parse_result.detection_metadata.detection_classes) + KEY_DETECTION_METADATA: { + KEY_DETECTION_CLASSES: list( + parse_result.detection_metadata.detection_classes + ) }, - "video_metadata": parse_result.video_metadata.to_dict(), + KEY_VIDEO_METADATA: parse_result.video_metadata.to_dict(), } return metadata diff --git a/OTAnalytics/plugin_parser/feathers_parser.py b/OTAnalytics/plugin_parser/feathers_parser.py index a39204ac6..76a7c2c17 100644 --- a/OTAnalytics/plugin_parser/feathers_parser.py +++ b/OTAnalytics/plugin_parser/feathers_parser.py @@ -29,6 +29,9 @@ ) from OTAnalytics.plugin_parser.convert_ottrk_to_feathers import ( FEATHER_FILETYPE, + KEY_DETECTION_CLASSES, + KEY_DETECTION_METADATA, + KEY_VIDEO_METADATA, METADATA_SUFFIX, convert_ottrk_to_feather, ) @@ -162,7 +165,7 @@ def parse(self, file: Path) -> TrackParseResult: if not file.exists(): raise FileNotFoundError(f"Feather file not found: {file}") # Construct metadata file path - metadata_file = file.parent / f"{file.stem}_metadata.json" + metadata_file = file.parent / f"{file.stem}{METADATA_SUFFIX}" if not metadata_file.exists(): raise FileNotFoundError(f"Metadata file not found: {metadata_file}") @@ -179,11 +182,11 @@ def parse(self, file: Path) -> TrackParseResult: ) # Parse video metadata - video_metadata = self._parse_video_metadata(metadata["video_metadata"]) + video_metadata = self._parse_video_metadata(metadata[KEY_VIDEO_METADATA]) # Parse detection metadata detection_metadata = self._parse_detection_metadata( - metadata["detection_metadata"] + metadata[KEY_DETECTION_METADATA] ) return TrackParseResult(tracks, detection_metadata, video_metadata) @@ -228,5 +231,5 @@ def _parse_detection_metadata(self, metadata: dict) -> DetectionMetadata: Returns: DetectionMetadata: Parsed detection metadata object """ - detection_classes = frozenset(metadata["detection_classes"]) + detection_classes = frozenset(metadata[KEY_DETECTION_CLASSES]) return DetectionMetadata(detection_classes) From 67804c67aa533f8d0c9a0d219ae723d2e37016fa Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Thu, 2 Oct 2025 20:34:59 +0200 Subject: [PATCH 113/128] Refactor: Introduce constants for recorded start date in test metadata and add corresponding assertions in `test_feathers_parser.py`. --- .../plugin_parser/test_feathers_parser.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/OTAnalytics/plugin_parser/test_feathers_parser.py b/tests/OTAnalytics/plugin_parser/test_feathers_parser.py index a44e73aa2..0e15fd67f 100644 --- a/tests/OTAnalytics/plugin_parser/test_feathers_parser.py +++ b/tests/OTAnalytics/plugin_parser/test_feathers_parser.py @@ -3,7 +3,7 @@ """ import tempfile -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any from unittest.mock import Mock, patch @@ -58,6 +58,12 @@ def sample_df() -> pd.DataFrame: ) +GIVEN_RECORDED_START_DATE = 1672574400.0 # 2022-01-01T10:00:00 as timestamp +EXPECTED_RECORDED_START_DATE = datetime.fromtimestamp( + GIVEN_RECORDED_START_DATE, tz=timezone.utc +) + + @pytest.fixture def sample_metadata() -> dict[str, Any]: """Create sample metadata for testing.""" @@ -65,7 +71,7 @@ def sample_metadata() -> dict[str, Any]: "detection_metadata": {"detection_classes": ["car", "truck", "bus"]}, "video_metadata": { "path": "test_video.mp4", - "recorded_start_date": 1672574400.0, # 2023-01-01T10:00:00 as timestamp + "recorded_start_date": GIVEN_RECORDED_START_DATE, "recorded_fps": 30.0, "number_of_frames": 900, "expected_duration": 30.0, @@ -165,6 +171,7 @@ def test_parse_success( video_metadata = result.video_metadata assert isinstance(video_metadata, VideoMetadata) assert video_metadata.path == "test_video.mp4" + assert video_metadata.start == EXPECTED_RECORDED_START_DATE assert video_metadata.recorded_fps == 30.0 assert video_metadata.number_of_frames == 900 assert video_metadata.actual_fps == 29.97 @@ -191,7 +198,7 @@ def test_parse_video_metadata_minimal(self, parser: FeathersParser) -> None: """Test parsing video metadata with minimal required fields.""" minimal_metadata = { "path": "video.mp4", - "recorded_start_date": 1672574400.0, # 2023-01-01T10:00:00 as timestamp + "recorded_start_date": GIVEN_RECORDED_START_DATE, "recorded_fps": 30.0, "number_of_frames": 900, } @@ -200,6 +207,7 @@ def test_parse_video_metadata_minimal(self, parser: FeathersParser) -> None: assert isinstance(result, VideoMetadata) assert result.path == "video.mp4" + assert result.start == EXPECTED_RECORDED_START_DATE assert result.recorded_fps == 30.0 assert result.number_of_frames == 900 assert result.expected_duration is None @@ -211,7 +219,7 @@ def test_parse_video_metadata_with_optional_fields( """Test parsing video metadata with all optional fields.""" full_metadata = { "path": "video.mp4", - "recorded_start_date": 1672574400.0, # 2023-01-01T10:00:00 as timestamp + "recorded_start_date": GIVEN_RECORDED_START_DATE, "recorded_fps": 30.0, "number_of_frames": 900, "expected_duration": 30.0, @@ -226,6 +234,7 @@ def test_parse_video_metadata_with_optional_fields( assert result.number_of_frames == 900 assert result.expected_duration == timedelta(seconds=30.0) assert result.actual_fps == 29.97 + assert result.start == EXPECTED_RECORDED_START_DATE def test_parse_detection_metadata(self, parser: FeathersParser) -> None: """Test parsing detection metadata.""" From e3191c3f571c7cc8c331d62b47747da9de1dcdab Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 3 Oct 2025 00:19:27 +0200 Subject: [PATCH 114/128] Refactor: Streamline unpacking track IDs by using set comprehension for `polars_track_id_set.py`. --- OTAnalytics/plugin_datastore/polars_track_id_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_id_set.py b/OTAnalytics/plugin_datastore/polars_track_id_set.py index 204be6e7b..7bf6936f8 100644 --- a/OTAnalytics/plugin_datastore/polars_track_id_set.py +++ b/OTAnalytics/plugin_datastore/polars_track_id_set.py @@ -59,7 +59,7 @@ def union(self, other: "TrackIdSet") -> "TrackIdSet": return PolarsTrackIdSet(combined_series) else: # Convert other to strings and combine - other_strings = [unpack(track_id) for track_id in other] + other_strings = {unpack(track_id) for track_id in other} other_series = pl.Series(other_strings, dtype=pl.String) combined_series = pl.concat([self._series, other_series]).unique().sort() return PolarsTrackIdSet(combined_series) From e50b2feddb90085b982e61f20d520b97f6053a4e Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Fri, 3 Oct 2025 17:29:52 +0200 Subject: [PATCH 115/128] Remove redundant import `EmptyTrackIdSet` from `polars_track_store.py`. --- OTAnalytics/plugin_datastore/polars_track_store.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 4fafe54c7..c82079f38 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -783,8 +783,6 @@ def revert_cuts_for( def _get_existing_track_ids(self, track_ids: TrackIdSet) -> TrackIdSet: if self._dataset.is_empty(): - from OTAnalytics.domain.track_dataset.track_dataset import EmptyTrackIdSet - return EmptyTrackIdSet() converted_ids = [unpack(track_id) for track_id in track_ids] From 00476f377e320af4b5d3a49a17f2ed9ed9c718b1 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Sat, 11 Oct 2025 13:03:21 +0200 Subject: [PATCH 116/128] Fix intersection calculation --- .../track_geometry_store/polars_geometry_store.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index d4792a1e3..781ec4ae3 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -236,8 +236,8 @@ def calculate_intersection_parameters( [ # Calculate denominator for intersection formula ( - pl.col("line_dx") * pl.col("seg_dy") - - pl.col("line_dy") * pl.col("seg_dx") + -pl.col("line_dx") * pl.col("seg_dy") + + pl.col("line_dy") * pl.col("seg_dx") ).alias(DENOMINATOR), ] ) @@ -255,8 +255,8 @@ def calculate_intersection_parameters( pl.when(pl.col(NON_PARALLEL)) .then( ( - (pl.col("seg_x1") - line_x1) * pl.col("seg_dy") - - (pl.col("seg_y1") - line_y1) * pl.col("seg_dx") + -(pl.col("seg_x1") - line_x1) * pl.col("seg_dy") + + (pl.col("seg_y1") - line_y1) * pl.col("seg_dx") ) / pl.col(DENOMINATOR) ) @@ -265,8 +265,8 @@ def calculate_intersection_parameters( pl.when(pl.col(NON_PARALLEL)) .then( ( - (pl.col("seg_x1") - line_x1) * pl.col("line_dy") - - (pl.col("seg_y1") - line_y1) * pl.col("line_dx") + pl.col("line_dx") * (pl.col("seg_y1") - line_y1) + - pl.col("line_dy") * (pl.col("seg_x1") - line_x1) ) / pl.col(DENOMINATOR) ) From 870d10d88fda6224241acca9cdebdd3901695fa2 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 13 Oct 2025 12:43:09 +0200 Subject: [PATCH 117/128] Fix direction vector calculation in PolarsGeometryDataset --- .../track_geometry_store/polars_geometry_store.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 781ec4ae3..7b3bd7ece 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -682,8 +682,8 @@ def create_events( pl.col(END_X).alias(EVENT_COORDINATE_X), pl.col(END_Y).alias(EVENT_COORDINATE_Y), pl.lit(EventType.SECTION_ENTER.value).alias(event.EVENT_TYPE), - (pl.col(PREVIOUS_Y) - pl.col(PREVIOUS_X)).alias(DIRECTION_VECTOR_X), - (pl.col(CURRENT_Y) - pl.col(CURRENT_X)).alias(DIRECTION_VECTOR_Y), + (pl.col(CURRENT_X) - pl.col(PREVIOUS_X)).alias(DIRECTION_VECTOR_X), + (pl.col(CURRENT_Y) - pl.col(PREVIOUS_Y)).alias(DIRECTION_VECTOR_Y), pl.col(END_VIDEO_NAME).alias(event.VIDEO_NAME), ( pl.col(START_OCCURRENCE) @@ -705,8 +705,10 @@ def create_events( .with_columns( [ ( - pl.col(DIRECTION_VECTOR_X) ** 2 - + pl.col(DIRECTION_VECTOR_Y) ** 2 + ( + pl.col(DIRECTION_VECTOR_X) ** 2 + + pl.col(DIRECTION_VECTOR_Y) ** 2 + ).sqrt() ).alias(MAGNITUDE), ] ) From cf67bbaa35d1547a71e98194f729cfe291b5ff1d Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 14 Oct 2025 16:54:34 +0200 Subject: [PATCH 118/128] Add interpolated columns to output --- .../application/export_formats/event_list.py | 1 + .../eventlist_exporter/eventlist_exporter.py | 61 ++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/application/export_formats/event_list.py b/OTAnalytics/application/export_formats/event_list.py index ec4634006..3c8953bbb 100644 --- a/OTAnalytics/application/export_formats/event_list.py +++ b/OTAnalytics/application/export_formats/event_list.py @@ -18,6 +18,7 @@ DIRECTION_VECTOR_Y = "vector_px_y" VIDEO_NAME = event.VIDEO_NAME INTERPOLATED_OCCURRENCE = event.INTERPOLATED_OCCURRENCE +INTERPOLATED_EVENT_COORDINATE = event.INTERPOLATED_EVENT_COORDINATE INTERPOLATED_EVENT_COORDINATE_X = "interpolated_coordinate_px_x" INTERPOLATED_EVENT_COORDINATE_Y = "interpolated_coordinate_px_y" diff --git a/OTAnalytics/plugin_prototypes/eventlist_exporter/eventlist_exporter.py b/OTAnalytics/plugin_prototypes/eventlist_exporter/eventlist_exporter.py index 959e892e7..8424bf76c 100644 --- a/OTAnalytics/plugin_prototypes/eventlist_exporter/eventlist_exporter.py +++ b/OTAnalytics/plugin_prototypes/eventlist_exporter/eventlist_exporter.py @@ -5,6 +5,26 @@ from OTAnalytics.application.config import DEFAULT_EVENTLIST_FILE_TYPE from OTAnalytics.application.datastore import EventListParser from OTAnalytics.application.export_formats import event_list +from OTAnalytics.application.export_formats.event_list import ( + DIRECTION_VECTOR_X, + DIRECTION_VECTOR_Y, + EVENT_COORDINATE_X, + EVENT_COORDINATE_Y, + EVENT_TYPE, + FRAME_NUMBER, + HOSTNAME, + INTERPOLATED_EVENT_COORDINATE_X, + INTERPOLATED_EVENT_COORDINATE_Y, + INTERPOLATED_OCCURRENCE, + OCCURRENCE, + OCCURRENCE_DATE, + OCCURRENCE_TIME, + ROAD_USER_ID, + ROAD_USER_TYPE, + SECTION_ID, + SECTION_NAME, + VIDEO_NAME, +) from OTAnalytics.application.export_formats.export_mode import INITIAL_MERGE from OTAnalytics.application.logger import logger from OTAnalytics.application.use_cases.export_events import ( @@ -24,6 +44,30 @@ OTC_CSV_FORMAT_NAME = "CSV (OpenTrafficCam)" OTC_OTEVENTS_FORMAT_NAME = "OTEvents (OpenTrafficCam)" +OCCURRENCE_SEC = f"{OCCURRENCE}_sec" + +EXPORT_COLUMNS = [ + ROAD_USER_ID, + ROAD_USER_TYPE, + HOSTNAME, + OCCURRENCE, + FRAME_NUMBER, + SECTION_ID, + EVENT_TYPE, + VIDEO_NAME, + OCCURRENCE_SEC, + EVENT_COORDINATE_X, + EVENT_COORDINATE_Y, + DIRECTION_VECTOR_X, + DIRECTION_VECTOR_Y, + SECTION_NAME, + OCCURRENCE_DATE, + OCCURRENCE_TIME, + INTERPOLATED_OCCURRENCE, + INTERPOLATED_EVENT_COORDINATE_X, + INTERPOLATED_EVENT_COORDINATE_Y, +] + class EventListDataFrameBuilder: def __init__(self, events: Iterable[Event], sections: Iterable[Section]): @@ -39,7 +83,7 @@ def build(self) -> pd.DataFrame: self._split_columns_with_lists() self._add_section_names() self._add_detailed_date_time_columns() - return self._df + return self._df.loc[:, EXPORT_COLUMNS] def _add_detailed_date_time_columns(self) -> None: occurrence_column = pd.to_datetime(self._df[event_list.OCCURRENCE]) @@ -69,8 +113,21 @@ def _split_columns_with_lists(self) -> None: self._df[event_list.DIRECTION_VECTOR].tolist(), index=self._df.index ) ) + self._df[ + [ + event_list.INTERPOLATED_EVENT_COORDINATE_X, + event_list.INTERPOLATED_EVENT_COORDINATE_Y, + ] + ] = pd.DataFrame( + self._df[event_list.INTERPOLATED_EVENT_COORDINATE].tolist(), + index=self._df.index, + ) self._df = self._df.drop( - columns=[event_list.EVENT_COORDINATE, event_list.DIRECTION_VECTOR] + columns=[ + event_list.EVENT_COORDINATE, + event_list.DIRECTION_VECTOR, + event_list.INTERPOLATED_EVENT_COORDINATE, + ] ) def _add_section_names(self) -> None: From bc3636836732dc7efc0121f41beaac7a60c5273e Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Tue, 14 Oct 2025 16:54:56 +0200 Subject: [PATCH 119/128] Fix wrong columns and extract hostname --- .../track_geometry_store/polars_geometry_store.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index 7b3bd7ece..caa256d9e 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -673,14 +673,14 @@ def create_events( [ pl.col(TRACK_ID).alias(event.ROAD_USER_ID), pl.col(TRACK_CLASSIFICATION).alias(event.ROAD_USER_TYPE), - pl.col(END_VIDEO_NAME).alias( - event.HOSTNAME - ), # TODO extract hostname + pl.col(END_VIDEO_NAME) + .str.extract(event.FILE_NAME_PATTERN) + .alias(event.HOSTNAME), pl.col(END_OCCURRENCE).alias(event.OCCURRENCE), pl.col(END_FRAME).alias(event.FRAME_NUMBER), pl.col(SECTION_ID).alias(event.SECTION_ID), - pl.col(END_X).alias(EVENT_COORDINATE_X), - pl.col(END_Y).alias(EVENT_COORDINATE_Y), + pl.col(CURRENT_X).alias(EVENT_COORDINATE_X), + pl.col(CURRENT_Y).alias(EVENT_COORDINATE_Y), pl.lit(EventType.SECTION_ENTER.value).alias(event.EVENT_TYPE), (pl.col(CURRENT_X) - pl.col(PREVIOUS_X)).alias(DIRECTION_VECTOR_X), (pl.col(CURRENT_Y) - pl.col(PREVIOUS_Y)).alias(DIRECTION_VECTOR_Y), From 86590eba13816b44580bb02283c18abed496b8f4 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 15 Oct 2025 13:52:14 +0200 Subject: [PATCH 120/128] Fix missing override when using filterd track dataset --- OTAnalytics/plugin_datastore/filter_polars_track_dataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OTAnalytics/plugin_datastore/filter_polars_track_dataset.py b/OTAnalytics/plugin_datastore/filter_polars_track_dataset.py index 5d1d39e38..d87a47903 100644 --- a/OTAnalytics/plugin_datastore/filter_polars_track_dataset.py +++ b/OTAnalytics/plugin_datastore/filter_polars_track_dataset.py @@ -64,6 +64,9 @@ def cut_with_section( dataset, original_track_ids = self._other.cut_with_section(section, offset) return self.wrap(dataset), original_track_ids + def ids_inside(self, sections: list[Section]) -> TrackIdSet: + return self._filter().ids_inside(sections) + @abstractmethod def _filter(self) -> PolarsTrackDataset: raise NotImplementedError From fed50c63fb122bff91289ef5bfda597989c93b95 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 15 Oct 2025 14:46:45 +0200 Subject: [PATCH 121/128] Compare csv files using polars and differ between events and counts --- tests/regression_otanalytics.py | 6 +-- tests/utils/assertions.py | 66 +++++++++++++++++---------------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/tests/regression_otanalytics.py b/tests/regression_otanalytics.py index a06b2c547..6daba9cce 100644 --- a/tests/regression_otanalytics.py +++ b/tests/regression_otanalytics.py @@ -8,7 +8,7 @@ from OTAnalytics.application.parser.flow_parser import FlowParser from OTAnalytics.plugin_cli.cli_application import OtAnalyticsCliApplicationStarter from OTAnalytics.plugin_parser.otvision_parser import OtFlowParser -from tests.utils.assertions import assert_two_files_equal_sorted +from tests.utils.assertions import assert_equal_count_files, assert_equal_event_files from tests.utils.builders.run_configuration import create_run_config @@ -274,7 +274,7 @@ def _execute_test( expected_events_file = Path(test_data_dir / save_name).with_suffix( ".events.csv" ) - assert_two_files_equal_sorted(actual_events_file, expected_events_file) + assert_equal_event_files(actual_events_file, expected_events_file) actual_counts_file = ( Path(test_data_tmp_dir / save_name) @@ -286,7 +286,7 @@ def _execute_test( .with_suffix(f".counts_{count_interval}min.csv") .absolute() ) - assert_two_files_equal_sorted(actual_counts_file, expected_counts_file) + assert_equal_count_files(actual_counts_file, expected_counts_file) def _run_otanalytics( self, diff --git a/tests/utils/assertions.py b/tests/utils/assertions.py index 8e5c52b89..c51e0368f 100644 --- a/tests/utils/assertions.py +++ b/tests/utils/assertions.py @@ -2,8 +2,14 @@ from typing import Iterable from unittest.mock import Mock +import polars as pl +from polars.testing import assert_frame_equal + +from OTAnalytics.application.export_formats import event_list +from OTAnalytics.application.export_formats.event_list import DATE_TIME_FORMAT from OTAnalytics.domain.track import Detection, Track from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset +from OTAnalytics.plugin_prototypes.eventlist_exporter import eventlist_exporter def assert_equal_detection_properties(actual: Detection, expected: Detection) -> None: @@ -62,39 +68,35 @@ def assert_track_dataset_has_tracks(dataset: TrackDataset, tracks: list[Track]) assert_equal_track_properties(actual, expected) -def assert_two_files_equal_sorted( +def assert_equal_event_files( actual_counts_file: Path, expected_counts_file: Path ) -> None: + actual = _read_event_csv(actual_counts_file) + expected = _read_event_csv(expected_counts_file) + assert_frame_equal(actual, expected) - with open(actual_counts_file, mode="r") as actual: - act_lines = sorted([line.split(",", 1)[1] for line in actual.readlines()]) - - with open(expected_counts_file, mode="r") as expected: - exp_lines = sorted([line.split(",", 1)[1] for line in expected.readlines()]) - - assert set(act_lines) == set(exp_lines), ( - f"Sets are not equal [act len: {len(act_lines)}, " - f"exp len {len(exp_lines)}]:\n" - f"actual invents (fst.): " - f"{([a for a in act_lines if a not in exp_lines] + [None])[0]}\n" - f"actual missing (fst.): " - f"{([e for e in exp_lines if e not in act_lines] + [None])[0]}\n" - f"actual invents (scnd): " - f"{([a for a in act_lines if a not in exp_lines] + [None] * 2)[1]}\n" - f"actual missing (scnd): " - f"{([e for e in exp_lines if e not in act_lines] + [None] * 2)[1]}\n" - f"actual invents (num): " - f"{len([a for a in act_lines if a not in exp_lines])}\n" - f"actual missing (num): " - f"{len([e for e in exp_lines if e not in act_lines])}\n" - ) - assert act_lines == exp_lines, ( - f"First difference of {actual_counts_file.name} ({len(act_lines)}) " - f"to expected {expected_counts_file.name} ({len(exp_lines)}):\n" - f"exp: " - f"{([e for e, a in zip(exp_lines, act_lines) if e != a] + [None])[0]}\n" - f"act: " - f"{([a for e, a in zip(exp_lines, act_lines) if e != a] + [None])[0]}\n" - f"\n" - ) +def assert_equal_count_files( + actual_counts_file: Path, expected_counts_file: Path +) -> None: + actual = _read_counts_csv(actual_counts_file) + expected = _read_counts_csv(expected_counts_file) + assert_frame_equal(actual, expected) + + +def _read_counts_csv(file: Path) -> pl.DataFrame: + return pl.read_csv(file) + + +def _read_event_csv(file: Path) -> pl.DataFrame: + data = pl.read_csv(file) + return ( + data.with_columns( + pl.col(event_list.INTERPOLATED_OCCURRENCE) + .str.to_datetime(DATE_TIME_FORMAT) + .dt.round("1ms") + ) + .with_columns(pl.col(event_list.INTERPOLATED_EVENT_COORDINATE_X).round(3)) + .with_columns(pl.col(event_list.INTERPOLATED_EVENT_COORDINATE_Y).round(3)) + .sort(by=eventlist_exporter.EXPORT_COLUMNS) + ) From 72eed3f5504086ea0d624a59c4ad7d8920db9ec4 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Wed, 15 Oct 2025 14:47:13 +0200 Subject: [PATCH 122/128] Fix track id for not cut tracks --- .../polars_geometry_store.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py index caa256d9e..da1fb62a7 100644 --- a/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py +++ b/OTAnalytics/plugin_datastore/track_geometry_store/polars_geometry_store.py @@ -62,6 +62,7 @@ MAGNITUDE = "magnitude" CUM_SUM = "cum_sum" ORDER = "order" +TRACK_ID_SUFFIX = f"{TRACK_ID}_suffix" SEGMENT_LENGTH_X = "segment_length_x" SEGMENT_LENGTH_Y = "segment_length_y" @@ -1393,6 +1394,16 @@ def track_ids_after_cut(self, section: Section) -> pl.DataFrame: ) temp_r = pl.concat([results, temp]) results = temp_r.sort(by=[TRACK_ID, ROW_ID, ORDER]) - return results.with_columns( - pl.col(TRACK_ID) + "_" + pl.col(CUM_SUM).cast(pl.Utf8).alias(TRACK_ID) - ).select([ROW_ID, TRACK_ID]) + cut_track_ids = ( + results.filter(pl.col(CUM_SUM) > 0).unique().select(TRACK_ID).to_series() + ) + return ( + results.with_columns( + pl.when(pl.col(TRACK_ID).is_in(cut_track_ids)) + .then("_" + pl.col(CUM_SUM).cast(pl.Utf8)) + .otherwise(pl.lit("").cast(pl.Utf8)) + .alias(TRACK_ID_SUFFIX) + ) + .with_columns((pl.col(TRACK_ID) + pl.col(TRACK_ID_SUFFIX)).alias(TRACK_ID)) + .select([ROW_ID, TRACK_ID]) + ) From e7ebc4db210d6cf6b68d20142609b77b83ea1d97 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Fri, 24 Oct 2025 09:12:12 +0200 Subject: [PATCH 123/128] OP#8596: Sort dataframe before processing --- OTAnalytics/plugin_datastore/polars_track_store.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index c82079f38..4fc3e0997 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -383,7 +383,11 @@ def from_dataframe( ) -> "PolarsTrackDataset": if tracks.is_empty(): return PolarsTrackDataset(track_geometry_factory) - tracks = tracks.drop(ROW_ID, strict=False).with_row_index(ROW_ID) + tracks = ( + tracks.drop(ROW_ID, strict=False) + .sort(by=[track.TRACK_ID, track.OCCURRENCE]) + .with_row_index(ROW_ID) + ) result = _assign_track_classification(tracks, calculator) return PolarsTrackDataset( track_geometry_factory, @@ -924,7 +928,8 @@ def _assign_track_classification( ) -> pl.DataFrame: dropped = _drop_track_classification(data) classification_per_track = calculator.calculate(dropped) - return dropped.join(classification_per_track, on=track.TRACK_ID, how="left") + result = dropped.join(classification_per_track, on=track.TRACK_ID, how="left") + return result def _drop_track_classification(data: pl.DataFrame) -> pl.DataFrame: From 8100e716a35bf883eff7698b53e2e36dc8ee2e75 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 27 Oct 2025 11:08:36 +0100 Subject: [PATCH 124/128] OP#8596: Compare events only as pandas data frames to be much faster. --- tests/regression_otanalytics.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/regression_otanalytics.py b/tests/regression_otanalytics.py index 298c66a79..951d4051e 100644 --- a/tests/regression_otanalytics.py +++ b/tests/regression_otanalytics.py @@ -8,11 +8,7 @@ from OTAnalytics.application.parser.flow_parser import FlowParser from OTAnalytics.plugin_cli.cli_application import OtAnalyticsCliApplicationStarter from OTAnalytics.plugin_parser.otvision_parser import OtFlowParser -from tests.utils.assertions import ( - assert_equal_count_files, - assert_equal_event_files, - assert_two_files_equal_sorted, -) +from tests.utils.assertions import assert_equal_count_files, assert_equal_event_files from tests.utils.builders.run_configuration import create_run_config @@ -285,7 +281,6 @@ def _execute_test( expected_events_file = Path(test_data_dir / save_name).with_suffix( ".events.csv" ) - assert_two_files_equal_sorted(actual_events_file, expected_events_file) assert_equal_event_files(actual_events_file, expected_events_file) def _run_otanalytics( From 1b26c44921a8f5c8955a3247f83b45b844b813c2 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 27 Oct 2025 11:09:19 +0100 Subject: [PATCH 125/128] OP#8596: Sort polars track dataset the same way as with previous parsers --- OTAnalytics/plugin_datastore/polars_track_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_store.py b/OTAnalytics/plugin_datastore/polars_track_store.py index 4fc3e0997..71d9908c5 100644 --- a/OTAnalytics/plugin_datastore/polars_track_store.py +++ b/OTAnalytics/plugin_datastore/polars_track_store.py @@ -385,7 +385,7 @@ def from_dataframe( return PolarsTrackDataset(track_geometry_factory) tracks = ( tracks.drop(ROW_ID, strict=False) - .sort(by=[track.TRACK_ID, track.OCCURRENCE]) + .sort(by=[track.TRACK_ID, track.OCCURRENCE, track.VIDEO_NAME, track.FRAME]) .with_row_index(ROW_ID) ) result = _assign_track_classification(tracks, calculator) From fc7f24910c8860ed23fc4ae337d2d601a2475fc3 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 27 Oct 2025 11:15:24 +0100 Subject: [PATCH 126/128] OP#8596: Compare datetime values with 1ms precision. --- tests/utils/assertions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/utils/assertions.py b/tests/utils/assertions.py index 6f744f424..d03350647 100644 --- a/tests/utils/assertions.py +++ b/tests/utils/assertions.py @@ -5,6 +5,7 @@ import polars as pl from polars.testing import assert_frame_equal +from OTAnalytics.application.export_formats.event_list import DATE_TIME_FORMAT from OTAnalytics.domain.track import Detection, Track from OTAnalytics.domain.track_dataset.track_dataset import TrackDataset from OTAnalytics.plugin_prototypes.eventlist_exporter import eventlist_exporter @@ -126,4 +127,9 @@ def _read_counts_csv(file: Path) -> pl.DataFrame: def _read_event_csv(file: Path) -> pl.DataFrame: data = pl.read_csv(file) - return data.sort(by=eventlist_exporter.EXPORT_COLUMNS) + return data.with_columns( + pl.col(eventlist_exporter.INTERPOLATED_OCCURRENCE) + .str.to_datetime(DATE_TIME_FORMAT) + .dt.round("1ms") + .alias(eventlist_exporter.INTERPOLATED_OCCURRENCE) + ).sort(by=eventlist_exporter.EXPORT_COLUMNS) From 3503baf1c6ae2c9fdf0216b08debf4fd78747c35 Mon Sep 17 00:00:00 2001 From: Lars Briem Date: Mon, 27 Oct 2025 11:15:35 +0100 Subject: [PATCH 127/128] OP#8596: Add debugging function --- tests/utils/assertions.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/utils/assertions.py b/tests/utils/assertions.py index d03350647..d01e73b9d 100644 --- a/tests/utils/assertions.py +++ b/tests/utils/assertions.py @@ -113,6 +113,31 @@ def assert_equal_event_files( assert_frame_equal(actual, expected) +def debugging_function_compare(actual: pl.DataFrame, expected: pl.DataFrame) -> None: + actual_small = actual.select( + [eventlist_exporter.ROAD_USER_ID, eventlist_exporter.FRAME_NUMBER] + ) + expected_small = ( + expected.with_columns( + pl.col(eventlist_exporter.ROAD_USER_ID).alias( + f"expected_{eventlist_exporter.ROAD_USER_ID}" + ) + ) + .with_columns( + pl.col(eventlist_exporter.FRAME_NUMBER).alias( + f"expected_{eventlist_exporter.FRAME_NUMBER}" + ) + ) + .drop([eventlist_exporter.ROAD_USER_ID, eventlist_exporter.FRAME_NUMBER]) + ) + temp = pl.concat([actual_small, expected_small], how="horizontal") + filtered = temp.filter( + pl.col(f"expected_{eventlist_exporter.FRAME_NUMBER}") + != pl.col(eventlist_exporter.FRAME_NUMBER) + ) + print(filtered) + + def assert_equal_count_files( actual_counts_file: Path, expected_counts_file: Path ) -> None: From ef009d3d9fa5527e5384689dd4aaf57447f56b84 Mon Sep 17 00:00:00 2001 From: Randy Seng <19281702+randy-seng@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:01:38 +0100 Subject: [PATCH 128/128] OP#8596: Sort intersected series after filtering in `polars_track_id_set.py`. --- OTAnalytics/plugin_datastore/polars_track_id_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OTAnalytics/plugin_datastore/polars_track_id_set.py b/OTAnalytics/plugin_datastore/polars_track_id_set.py index 7bf6936f8..ee0af9ce1 100644 --- a/OTAnalytics/plugin_datastore/polars_track_id_set.py +++ b/OTAnalytics/plugin_datastore/polars_track_id_set.py @@ -49,7 +49,7 @@ def intersection(self, other: "TrackIdSet") -> "TrackIdSet": other_strings = {unpack(track_id) for track_id in other} intersected_series = self._series.filter( self._series.is_in(list(other_strings)) - ) + ).sort() return PolarsTrackIdSet(intersected_series) def union(self, other: "TrackIdSet") -> "TrackIdSet":