|
| 1 | +"""Collapse duplicate CSV rows by putting them in a set of frozendicts. |
| 2 | +
|
| 3 | +Each row that csv.DictReader yields is a dict, which is unhashable and so can't |
| 4 | +go in a set. Freezing each row makes the whole deduplication one expression. |
| 5 | +
|
| 6 | +Watch orders 1002 and 1003: they survive as two entries each, but for different |
| 7 | +reasons. The 1002 rows differ only in fetched_at, while the 1003 rows also |
| 8 | +differ in amount. Ignoring fetched_at therefore merges 1002 and leaves 1003 |
| 9 | +split. That's a lesson about picking the fields that define identity, not a |
| 10 | +bug. |
| 11 | +
|
| 12 | +Run with Python 3.15 or later: |
| 13 | +
|
| 14 | + python dedupe_csv.py |
| 15 | +""" |
| 16 | + |
| 17 | +import csv |
| 18 | +import io |
| 19 | +from operator import itemgetter |
| 20 | + |
| 21 | +ORDERS = """\ |
| 22 | +order_id,customer,amount,fetched_at |
| 23 | +1001,Ada,250.00,2026-08-13T09:00:00 |
| 24 | +1002,Grace,80.50,2026-08-13T09:00:00 |
| 25 | +1001,Ada,250.00,2026-08-13T09:00:00 |
| 26 | +1003,Linus,42.00,2026-08-13T09:00:00 |
| 27 | +1002,Grace,80.50,2026-08-13T09:05:00 |
| 28 | +1003,Linus,99.00,2026-08-13T09:05:00 |
| 29 | +""" |
| 30 | + |
| 31 | + |
| 32 | +def main(): |
| 33 | + rows = list(csv.DictReader(io.StringIO(ORDERS))) |
| 34 | + unique_rows = {frozendict(row) for row in rows} |
| 35 | + |
| 36 | + print( |
| 37 | + f"Read {len(rows)} rows, kept {len(unique_rows)} after deduplication." |
| 38 | + ) |
| 39 | + for row in sorted(unique_rows, key=itemgetter("order_id", "fetched_at")): |
| 40 | + print( |
| 41 | + f" {row['order_id']} {row['customer']:<6} " |
| 42 | + f"{row['amount']:>7} {row['fetched_at']}" |
| 43 | + ) |
| 44 | + |
| 45 | + identity = itemgetter("order_id", "customer", "amount") |
| 46 | + by_order = { |
| 47 | + frozendict(zip(("order_id", "customer", "amount"), identity(row))) |
| 48 | + for row in rows |
| 49 | + } |
| 50 | + print(f"Ignoring fetched_at leaves {len(by_order)} orders.") |
| 51 | + |
| 52 | + |
| 53 | +if __name__ == "__main__": |
| 54 | + main() |
0 commit comments