|
| 1 | +"""Find dates and Unix timestamps that fall in a realistic, possibly real, range. |
| 2 | +
|
| 3 | +De-identified datasets usually shift dates, often far into the future. A date |
| 4 | +between `lookback_years` ago and today may be a real date, so the privacy check |
| 5 | +asks the submitter to confirm it was shifted. Format alone cannot tell shifted |
| 6 | +dates from real ones, so findings are warnings to review, not errors. |
| 7 | +
|
| 8 | +Recognized forms: |
| 9 | +- Year-first dates: 2023-01-13, 2023/01/13, 2023.01.13, with an optional time. |
| 10 | +- Day- or month-first dates with a four-digit year: 13/01/2023, 01-13-2023. |
| 11 | +- Month names: 13 Jan 2023, January 13, 2023, Jan 2023. |
| 12 | +- Unix seconds (9-10 digits) or milliseconds (12-13 digits), but only in |
| 13 | + columns whose name suggests a time (or that a layout names), because IDs and |
| 14 | + counters have the same shape. |
| 15 | +
|
| 16 | +Two-digit years, eight-digit compact dates (20230113), times with no date, and |
| 17 | +years on their own are not reported. |
| 18 | +""" |
| 19 | + |
| 20 | +import re |
| 21 | +from dataclasses import dataclass |
| 22 | +from datetime import date, datetime, timedelta, timezone |
| 23 | +from typing import Iterator, Optional |
| 24 | + |
| 25 | +from physionet.validate.config import validate_lookback_years |
| 26 | + |
| 27 | + |
| 28 | +_MONTH_NAME = ( |
| 29 | + r"(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?" |
| 30 | + r"|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\.?" |
| 31 | +) |
| 32 | +_MONTHS = {name: number for number, name in enumerate( |
| 33 | + ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"], start=1 |
| 34 | +)} |
| 35 | + |
| 36 | +_YEAR_FIRST = re.compile(r"(?<!\d)(\d{4})([-/.])(\d{1,2})\2(\d{1,2})(?!\d)") |
| 37 | +# A trailing period is allowed (sentence end), but not a decimal like 13.01.2023.5. |
| 38 | +_YEAR_LAST = re.compile(r"(?<!\d)(?<!\d\.)(\d{1,2})([-/.])(\d{1,2})\2(\d{4})(?!\d|\.\d)") |
| 39 | +_DAY_MONTH_YEAR = re.compile( |
| 40 | + rf"\b(\d{{1,2}})(?:st|nd|rd|th)?\s+{_MONTH_NAME},?\s+(\d{{4}})\b", re.IGNORECASE |
| 41 | +) |
| 42 | +_MONTH_DAY_YEAR = re.compile( |
| 43 | + rf"\b{_MONTH_NAME}\s+(\d{{1,2}})(?:st|nd|rd|th)?,?\s+(\d{{4}})\b", re.IGNORECASE |
| 44 | +) |
| 45 | +_MONTH_YEAR = re.compile(rf"\b{_MONTH_NAME},?\s+(\d{{4}})\b", re.IGNORECASE) |
| 46 | + |
| 47 | +# Cheap pre-check: a written date needs two separators between digits, or a word. |
| 48 | +_MAYBE_WRITTEN_DATE = re.compile(r"\d[-/.]\d{1,2}[-/.]\d|[a-z]{3}", re.IGNORECASE) |
| 49 | + |
| 50 | +_UNIX = re.compile(r"^([1-9]\d{8,9}|[1-9]\d{11,12})(?:\.\d+)?$") |
| 51 | +_TIME_LIKE_COLUMN = re.compile(r"time|date|epoch|(?:^|[^a-z])ts(?:[^a-z]|$)", re.IGNORECASE) |
| 52 | + |
| 53 | + |
| 54 | +def _make_date(year: int, month: int, day: int) -> Optional[date]: |
| 55 | + try: |
| 56 | + return date(year, month, day) |
| 57 | + except ValueError: |
| 58 | + return None |
| 59 | + |
| 60 | + |
| 61 | +def _month(name: str) -> int: |
| 62 | + return _MONTHS[name.lower()[:3]] |
| 63 | + |
| 64 | + |
| 65 | +def dates_in_text(text: str) -> Iterator[date]: |
| 66 | + """Yield every calendar date written in text.""" |
| 67 | + if not _MAYBE_WRITTEN_DATE.search(text) or not any(char.isdigit() for char in text): |
| 68 | + return |
| 69 | + for match in _YEAR_FIRST.finditer(text): |
| 70 | + found = _make_date(int(match[1]), int(match[3]), int(match[4])) |
| 71 | + if found: |
| 72 | + yield found |
| 73 | + for match in _YEAR_LAST.finditer(text): |
| 74 | + first, second, year = int(match[1]), int(match[3]), int(match[4]) |
| 75 | + # Ambiguous order: accept month-first or day-first, whichever is valid. |
| 76 | + found = _make_date(year, first, second) or _make_date(year, second, first) |
| 77 | + if found: |
| 78 | + yield found |
| 79 | + if not any(char.isalpha() for char in text): |
| 80 | + return |
| 81 | + full_date_spans = [] |
| 82 | + for match in _DAY_MONTH_YEAR.finditer(text): |
| 83 | + found = _make_date(int(match[3]), _month(match[2]), int(match[1])) |
| 84 | + if found: |
| 85 | + full_date_spans.append(match.span()) |
| 86 | + yield found |
| 87 | + for match in _MONTH_DAY_YEAR.finditer(text): |
| 88 | + found = _make_date(int(match[3]), _month(match[1]), int(match[2])) |
| 89 | + if found: |
| 90 | + full_date_spans.append(match.span()) |
| 91 | + yield found |
| 92 | + for match in _MONTH_YEAR.finditer(text): |
| 93 | + # "5 May 2019" also contains "May 2019"; count only the full date. |
| 94 | + if any(start < match.end() and match.start() < end for start, end in full_date_spans): |
| 95 | + continue |
| 96 | + found = _make_date(int(match[2]), _month(match[1]), 1) |
| 97 | + if found: |
| 98 | + yield found |
| 99 | + |
| 100 | + |
| 101 | +def unix_timestamp_date(value: str) -> Optional[date]: |
| 102 | + """Return the UTC date of Unix seconds or milliseconds, if value is one.""" |
| 103 | + match = _UNIX.match(value.strip()) |
| 104 | + if not match: |
| 105 | + return None |
| 106 | + digits = match[1] |
| 107 | + seconds = int(digits) / 1000 if len(digits) > 10 else int(digits) |
| 108 | + try: |
| 109 | + return datetime.fromtimestamp(seconds, tz=timezone.utc).date() |
| 110 | + except (OverflowError, OSError, ValueError): |
| 111 | + return None |
| 112 | + |
| 113 | + |
| 114 | +def is_time_like_column(name: Optional[str]) -> bool: |
| 115 | + """Guess from a column name whether it holds times (e.g. charttime, ts, date).""" |
| 116 | + return bool(name) and bool(_TIME_LIKE_COLUMN.search(name)) |
| 117 | + |
| 118 | + |
| 119 | +@dataclass |
| 120 | +class DateRange: |
| 121 | + """Earliest and latest realistic dates seen in one place.""" |
| 122 | + |
| 123 | + earliest: date |
| 124 | + latest: date |
| 125 | + |
| 126 | + def add(self, found: date) -> None: |
| 127 | + self.earliest = min(self.earliest, found) |
| 128 | + self.latest = max(self.latest, found) |
| 129 | + |
| 130 | + def describe(self) -> str: |
| 131 | + if self.earliest == self.latest: |
| 132 | + return self.earliest.isoformat() |
| 133 | + return f"{self.earliest.isoformat()} to {self.latest.isoformat()}" |
| 134 | + |
| 135 | + |
| 136 | +class RealisticDateFinder: |
| 137 | + """Collect dates between lookback_years ago and today.""" |
| 138 | + |
| 139 | + def __init__(self, lookback_years: int, today: Optional[date] = None): |
| 140 | + validate_lookback_years(lookback_years) |
| 141 | + today = today or datetime.now(timezone.utc).date() |
| 142 | + self.lookback_years = lookback_years |
| 143 | + self.start = date(today.year - lookback_years, 1, 1) |
| 144 | + self.end = today + timedelta(days=1) # Allow for time-zone differences. |
| 145 | + |
| 146 | + def in_range(self, found: date) -> bool: |
| 147 | + return self.start <= found <= self.end |
| 148 | + |
| 149 | + def scan(self, value: str, unix_allowed: bool, ranges: dict, key: str) -> None: |
| 150 | + """Record realistic dates in value under key in ranges.""" |
| 151 | + candidates = list(dates_in_text(value)) |
| 152 | + if unix_allowed: |
| 153 | + unix_date = unix_timestamp_date(value) |
| 154 | + if unix_date: |
| 155 | + candidates.append(unix_date) |
| 156 | + for found in candidates: |
| 157 | + if not self.in_range(found): |
| 158 | + continue |
| 159 | + if key in ranges: |
| 160 | + ranges[key].add(found) |
| 161 | + else: |
| 162 | + ranges[key] = DateRange(found, found) |
0 commit comments