creating own datetime parser

Author: solegalli

feature_engine/datetime/datetime.py

@@ -5,6 +5,7 @@
 import pandas as pd
 from pandas.api.types import is_datetime64_any_dtype as is_datetime
 from pandas.api.types import is_numeric_dtype as is_numeric
+from pandas.api.types import is_object_dtype, is_string_dtype
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.utils.validation import check_is_fitted
 
@@ -38,13 +39,41 @@
     FEATURES_SUFFIXES,
     FEATURES_SUPPORTED,
 )
-from feature_engine.variable_handling._variable_type_checks import (
-    _is_categorical_and_is_datetime,
-)
 from feature_engine.variable_handling.check_variables import check_datetime_variables
 from feature_engine.variable_handling.find_variables import find_datetime_variables
 
 
+def _index_is_categorical_and_is_datetime(index: pd.Index) -> bool:
+    # This file is fully pandas-based (it casts with `pd.to_datetime` during
+    # `transform()`), and this check only ever runs against a pandas `Index`
+    # (narwhals has no `Index` concept), so it stays pandas-only rather than
+    # routing through the narwhals-based `_variable_type_checks` helpers.
+    is_object = is_object_dtype(index) or is_string_dtype(index)
+
+    if isinstance(index.dtype, pd.CategoricalDtype):
+        categories_are_numeric = is_numeric(index.categories)
+        if categories_are_numeric:
+            return False
+        try:
+            return is_datetime(pd.to_datetime(index, utc=True))
+        except Exception:
+            return False
+
+    elif is_object:
+        try:
+            is_convertible_to_num = is_numeric(pd.to_numeric(index))
+        except (ValueError, TypeError):
+            is_convertible_to_num = False
+        if is_convertible_to_num:
+            return False
+        try:
+            return is_datetime(pd.to_datetime(index, utc=True))
+        except Exception:
+            return False
+
+    return False
+
+
 @Substitution(
     return_empty=_return_empty_docstring,
     feature_names_in_=_feature_names_in_docstring,
@@ -265,7 +294,8 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
             if not (
                 is_datetime(X.index)
                 or (
-                    not is_numeric(X.index) and _is_categorical_and_is_datetime(X.index)
+                    not is_numeric(X.index)
+                    and _index_is_categorical_and_is_datetime(X.index)
                 )
             ):
                 raise TypeError("The dataframe index is not datetime.")

feature_engine/variable_handling/_variable_type_checks.py

@@ -1,127 +1,94 @@
+from datetime import date, datetime
+
 import narwhals as nw
-import pandas as pd
-from pandas.api.types import is_object_dtype, is_string_dtype
-from pandas.core.dtypes.common import is_datetime64_any_dtype as is_datetime
-from pandas.core.dtypes.common import is_numeric_dtype as is_numeric
+from dateutil.parser import parse as _dateutil_parse
 
 # ---------------------------------------------------------------------------
-# pandas-only implementation.
+# narwhals implementation, used for every backend (pandas, polars, etc.)
 #
-# These functions rely on pandas' flexible, dateutil-backed `pd.to_datetime`
-# string guessing and on pandas' `object` dtype (which, unlike any narwhals
-# dtype, can hold arbitrary non-string Python objects). Neither has a
-# polars/narwhals equivalent, so they are kept exactly as they were before the
-# narwhals migration and are only ever called on pandas input. See the `_nw_*`
-# functions below for the polars/narwhals-backend equivalents.
+# Flexible date-string recognition (e.g. "01-Jan-2010", "10/11/12", not just
+# ISO-8601) is implemented directly on top of `dateutil` - the same library
+# pandas.to_datetime delegates to internally for this - so it works
+# identically regardless of the underlying dataframe library.
 # ---------------------------------------------------------------------------
 
 
-def is_object(s) -> bool:
-    return is_object_dtype(s) or is_string_dtype(s)
-
-
-def _is_categorical_and_is_not_datetime(column: pd.Series) -> bool:
-    # check for datetime only if the type of the categories is not numeric
-    # because pd.to_datetime throws an error when it is an integer
-    if isinstance(column.dtype, pd.CategoricalDtype):
-        is_cat = _is_categories_num(column) or not _is_convertible_to_dt(column)
-
-    # check for datetime only if object cannot be cast as numeric because
-    # if it could pd.to_datetime would convert it to datetime regardless
-    elif is_object(column):
-        is_cat = _is_convertible_to_num(column) or not _is_convertible_to_dt(column)
-
-    else:
-        is_cat = False
-
-    return is_cat
+def _nw_is_date_or_datetime(dtype) -> bool:
+    # nw.selectors.datetime() only matches Datetime, not Date, so this needs
+    # its own explicit check.
+    return isinstance(dtype, (nw.Date, nw.Datetime))
 
 
-def _is_categories_num(column: pd.Series) -> bool:
-    return is_numeric(column.dtype.categories)
+_DATE_PARSE_DEFAULT_1 = datetime(1, 1, 1, 1, 1, 1)
+_DATE_PARSE_DEFAULT_2 = datetime(2, 2, 2, 2, 2, 2)
+_DATETIME_FIELDS = ("year", "month", "day", "hour", "minute", "second")
 
 
-def _is_convertible_to_dt(column: pd.Series) -> bool:
+def _looks_like_date_string(value: str) -> bool:
+    # dateutil.parser.parse() fills in any date/time component that isn't
+    # present in the string from a `default` datetime, so a bare number like
+    # "20" "parses" successfully as day=20 - it would wrongly be treated as a
+    # date. Parsing twice, with two defaults that differ in every field,
+    # reveals which fields were actually present in the string: those are the
+    # fields that agree between the two parses. Requiring at least 2 fields to
+    # be corroborated this way rejects bare numbers while still accepting real
+    # dates (including non-ISO formats like "01-Jan-2010") and bare times
+    # (like "21:45:23").
     try:
-        var = pd.to_datetime(column, utc=True)
-        return is_datetime(var)
-    except Exception:
+        first = _dateutil_parse(value, default=_DATE_PARSE_DEFAULT_1)
+        second = _dateutil_parse(value, default=_DATE_PARSE_DEFAULT_2)
+    except (ValueError, OverflowError, TypeError):
         return False
 
-
-def _is_convertible_to_num(column: pd.Series) -> bool:
-    try:
-        ser = pd.to_numeric(column)
-    except (ValueError, TypeError):
-        ser = column
-    return is_numeric(ser)
-
-
-def _is_categorical_and_is_datetime(column: pd.Series) -> bool:
-    # check for datetime only if the type of the categories is not numeric
-    # because pd.to_datetime throws an error when it is an integer
-    if isinstance(column.dtype, pd.CategoricalDtype):
-        is_dt = not _is_categories_num(column) and _is_convertible_to_dt(column)
-
-    # check for datetime only if object cannot be cast as numeric because
-    # if it could pd.to_datetime would convert it to datetime regardless
-    elif is_object(column):
-        is_dt = not _is_convertible_to_num(column) and _is_convertible_to_dt(column)
-
-    else:
-        is_dt = False
-
-    return is_dt
-
-
-# ---------------------------------------------------------------------------
-# narwhals implementation, used for every backend other than pandas (polars,
-# in practice).
-#
-# narwhals has no lenient/"try" cast (no `strict=False`, unlike raw polars)
-# and its `str.to_datetime()` requires ISO-8601 or an explicit `format=` - it
-# cannot reproduce pandas' dateutil-based guessing. So a string column such as
-# "01-Jan-2010" or "10/11/12" is not auto-detected as datetime for polars,
-# even though it is for pandas. ISO-8601 strings and native Date/Datetime
-# columns are detected correctly. Users can always pass `variables` explicitly
-# to sidestep this.
-# ---------------------------------------------------------------------------
-
-
-def _nw_is_date_or_datetime(dtype) -> bool:
-    # nw.selectors.datetime() only matches Datetime, not Date, so this needs
-    # its own explicit check.
-    return isinstance(dtype, (nw.Date, nw.Datetime))
+    corroborated = sum(
+        1 for attr in _DATETIME_FIELDS if getattr(first, attr) == getattr(second, attr)
+    )
+    return corroborated >= 2
 
 
 def _nw_is_convertible_to_num(s: "nw.Series") -> bool:
+    values = s.drop_nulls().to_list()
+    if not values:
+        return False
     try:
-        s.cast(nw.String()).cast(nw.Float64())
-    except Exception:
+        for value in values:
+            float(value)
+    except (ValueError, TypeError):
         return False
     return True
 
 
 def _nw_is_convertible_to_dt(s: "nw.Series") -> bool:
-    try:
-        s.cast(nw.String()).str.to_datetime()
-    except Exception:
+    values = s.drop_nulls().to_list()
+    if not values:
         return False
+    for value in values:
+        if isinstance(value, (date, datetime)):
+            continue
+        if not _looks_like_date_string(str(value)):
+            return False
     return True
 
 
+def _nw_categories_are_numeric(s: "nw.Series") -> bool:
+    return s.cat.get_categories().dtype.is_numeric()
+
+
 def _nw_is_categorical_and_is_not_datetime(s: "nw.Series") -> bool:
     if isinstance(s.dtype, nw.Enum):
         # an explicit, user-defined category set is an unambiguous categorical
         # signal, unlike a generic string column, so skip the datetime check
         return True
 
     if isinstance(s.dtype, nw.Categorical):
-        # polars categorical categories are always string-backed, unlike
-        # pandas' pd.Categorical, which can have numeric categories
-        return not _nw_is_convertible_to_dt(s)
-
-    if isinstance(s.dtype, nw.String):
+        # check for datetime only if the categories are not numeric, because
+        # a numeric-backed categorical (pandas-only - polars categories are
+        # always string-backed) can never hold dates
+        return _nw_categories_are_numeric(s) or not _nw_is_convertible_to_dt(s)
+
+    if isinstance(s.dtype, (nw.String, nw.Object)):
+        # check for datetime only if the column cannot be cast as numeric,
+        # because if it could, it would be a numeric column, not a date
         return _nw_is_convertible_to_num(s) or not _nw_is_convertible_to_dt(s)
 
     return False
@@ -132,9 +99,9 @@ def _nw_is_categorical_and_is_datetime(s: "nw.Series") -> bool:
         return False
 
     if isinstance(s.dtype, nw.Categorical):
-        return _nw_is_convertible_to_dt(s)
+        return not _nw_categories_are_numeric(s) and _nw_is_convertible_to_dt(s)
 
-    if isinstance(s.dtype, nw.String):
+    if isinstance(s.dtype, (nw.String, nw.Object)):
         return not _nw_is_convertible_to_num(s) and _nw_is_convertible_to_dt(s)
 
     return False

feature_engine/variable_handling/check_variables.py

@@ -5,14 +5,11 @@
 import narwhals as nw
 import narwhals.dependencies as nwd
 from narwhals.typing import IntoDataFrame
-from pandas.core.dtypes.common import is_numeric_dtype as is_numeric
 
 from feature_engine.variable_handling._variable_type_checks import (
-    _is_categorical_and_is_datetime,
     _nw_is_categorical_and_is_datetime,
     _nw_is_date_or_datetime,
 )
-from feature_engine.variable_handling.dtypes import DATETIME_TYPES
 
 Variables = Union[int, str, List[Union[str, int]]]
 
@@ -165,10 +162,9 @@ def check_datetime_variables(
 
     Notes
     -----
-    For pandas dataframes, string columns are parsed with pandas' flexible,
-    dateutil-backed date guessing. For polars (and other non-pandas dataframes),
-    only ISO-8601 strings and native `Date`/`Datetime` columns are recognised -
-    polars has no equivalent flexible guesser.
+    String columns are parsed with flexible, dateutil-backed date guessing, in
+    addition to ISO-8601 strings and native `Date`/`Datetime` columns,
+    regardless of the dataframe library backing `X`.
 
     Examples
     --------
@@ -187,27 +183,18 @@ def check_datetime_variables(
     if isinstance(variables, (str, int)):
         variables = [variables]
 
-    if nwd.is_pandas_dataframe(X):
-        # find non datetime variables, if any:
-        non_datetime_vars = []
-        for column in X[variables].select_dtypes(exclude=DATETIME_TYPES):
-            if is_numeric(X[column]) or not _is_categorical_and_is_datetime(
-                X[column]
-            ):
-                non_datetime_vars.append(column)
-    else:
-        sub_X = nw.from_native(X, eager_only=True).select(variables)
-        candidates = [
-            column
-            for column in sub_X.columns
-            if not _nw_is_date_or_datetime(sub_X.schema[column])
-        ]
-        non_datetime_vars = [
-            column
-            for column in candidates
-            if sub_X.schema[column].is_numeric()
-            or not _nw_is_categorical_and_is_datetime(sub_X[column])
-        ]
+    sub_X = nw.from_native(X, eager_only=True).select(variables)
+    candidates = [
+        column
+        for column in sub_X.columns
+        if not _nw_is_date_or_datetime(sub_X.schema[column])
+    ]
+    non_datetime_vars = [
+        column
+        for column in candidates
+        if sub_X.schema[column].is_numeric()
+        or not _nw_is_categorical_and_is_datetime(sub_X.get_column(column))
+    ]
 
     if len(non_datetime_vars) > 0:
         raise TypeError(