feat(import): skip duplicate records during CSV import

Author: LukasLJL

app/routes/api.py

@@ -3409,6 +3409,31 @@ def _cleanup_temp_file(path):
         pass
 
 
+def _is_duplicate(data_type, record, vehicle_id):
+    """Check whether a record with the same key fields already exists."""
+    if data_type == 'fuel_logs':
+        return db.session.query(FuelLog.id).filter_by(
+            vehicle_id=vehicle_id, date=record.date, odometer=record.odometer,
+        ).first() is not None
+    if data_type == 'expenses':
+        return db.session.query(Expense.id).filter_by(
+            vehicle_id=vehicle_id, date=record.date,
+            cost=record.cost, description=record.description,
+        ).first() is not None
+    if data_type == 'trips':
+        return db.session.query(Trip.id).filter_by(
+            vehicle_id=vehicle_id, date=record.date,
+            start_odometer=record.start_odometer,
+            end_odometer=record.end_odometer,
+        ).first() is not None
+    if data_type == 'charging_sessions':
+        return db.session.query(ChargingSession.id).filter_by(
+            vehicle_id=vehicle_id, date=record.date,
+            kwh_added=record.kwh_added,
+        ).first() is not None
+    return False
+
+
 def create_record(data_type, mapped_row, vehicle_id, user_id, date_format, user_date_format=None):
     """Create a model instance from a mapped CSV row."""
     date_val = parse_date_value(mapped_row.get('date', ''), date_format, user_date_format)
@@ -3662,6 +3687,7 @@ def csv_import_execute():
             rows = list(reader)
 
         imported = 0
+        skipped = 0
         errors = []
         max_errors = 50
 
@@ -3672,6 +3698,9 @@ def csv_import_execute():
                     mapped_row[field_name] = row.get(csv_col, '')
 
                 record = create_record(data_type, mapped_row, vehicle_id, current_user.id, date_format, current_user.date_format)
+                if _is_duplicate(data_type, record, vehicle_id):
+                    skipped += 1
+                    continue
                 db.session.add(record)
                 imported += 1
             except (ValueError, KeyError) as e:
@@ -3683,6 +3712,9 @@ def csv_import_execute():
         label = DATA_TYPE_LABELS.get(data_type, data_type)
         flash(_('CSV import complete: %(count)s %(label)s imported.') % {'count': imported, 'label': label.lower()}, 'success')
 
+        if skipped:
+            flash(_('%(count)s duplicate(s) skipped.') % {'count': skipped}, 'info')
+
         if errors:
             error_summary = f'{len(errors)} row(s) skipped due to errors.'
             if len(errors) <= 10: