feat: date detection

.gitignore

@@ -22,3 +22,6 @@ tests/output/
 *.jsonl.tar.gz
 
 .DS_Store
+
+# ignore temp excel files (maybe)
+~$*.xlsx

nuh_helper/date_shift/__init__.py

@@ -44,8 +44,14 @@ def __init__(self, page: str, row: int, col: int, col_name: str, val: str) -> No
 # >> pr 129 Exception goes here
 # << end of pr 129
 
-# >> pr 130 Exception goes here
-# << end of pr 130
+
+class HiddenDate(Exception):
+    def __init__(
+        self, sheet_name: str, row: int, col: int, value: str, found: datetime
+    ) -> None:
+        message = f"hidden date in [{sheet_name=}, {row}, {col}] {value=} // {found=}"
+        super().__init__(message)
+        self._message = message
 
 
 # >> pr 131 Exception goes here
@@ -566,6 +572,19 @@ def shift_excel_dates_inplace(
                 else:
                     cell.value = cast(Any, shifted.to_pydatetime())
 
+            # check for dates in non-date columns
+            for non_date_col_idx in range(1, 1 + (ws.max_column or 0)):
+                if non_date_col_idx in date_col_indices.values():
+                    continue
+
+                value = str(ws.cell(row=row_idx, column=non_date_col_idx).value)
+                import datefinder
+
+                for found in datefinder.find_dates(value):
+                    raise HiddenDate(
+                        sheet_name, row_idx, non_date_col_idx, value, found
+                    )
+
     wb.save(output_file)
     logger.info("Output written to '%s'", output_file)
 

pyproject.toml

@@ -8,6 +8,7 @@ requires-python = ">=3.13"
 dependencies = [
     "pandas>=3.0.5",
     "openpyxl>=3.1.0",
+    "datefinder>=1.0.0",
 ]
 
 [build-system]

tests/test_hidden_dates.py

@@ -0,0 +1,122 @@
+from pathlib import Path
+
+import pytest
+
+from nuh_helper import shift_excel_dates_inplace
+from nuh_helper.date_shift import (
+    HiddenDate,
+)
+
+
+def test_iso8601(tmp_path: Path) -> None:
+
+    source_file = Path(__file__).parent / "data/hidden_dates/iso8601.xlsx"
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
+
+    sheet_configs = {
+        "args": {
+            "patient_id_col": "ptid",
+            "date_columns": [
+                "dob",
+            ],
+            "header_row": 0,
+            "skip_rows_after_header": [],
+        },
+    }
+    with pytest.raises(HiddenDate) as info:
+        shift_excel_dates_inplace(
+            input_file=str(source_file),
+            output_file=str(output_path),
+            patient_sheet="args",
+            patient_id_col="ptid",
+            sheet_configs=sheet_configs,
+            min_shift_days=-20,
+            max_shift_days=-1,
+            seed=14333,
+            linking_table_path=str(linking_table_old),
+            linking_table_output=str(linking_table_out),
+        )
+
+    assert info.value._message == (
+        "hidden date in [sheet_name='args', 4, 3]"
+        + " value='hives on 2023-10-12'"
+        + " // found=datetime.datetime(2023, 10, 12, 0, 0)"
+    )
+
+
+def test_us_date(tmp_path: Path) -> None:
+
+    source_file = Path(__file__).parent / "data/hidden_dates/us_date.xlsx"
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
+
+    sheet_configs = {
+        "data": {
+            "patient_id_col": "ptid",
+            "date_columns": [
+                "dob",
+            ],
+            "header_row": 0,
+            "skip_rows_after_header": [],
+        },
+    }
+    with pytest.raises(HiddenDate) as info:
+        shift_excel_dates_inplace(
+            input_file=str(source_file),
+            output_file=str(output_path),
+            patient_sheet="data",
+            patient_id_col="ptid",
+            sheet_configs=sheet_configs,
+            min_shift_days=-20,
+            max_shift_days=-1,
+            seed=14333,
+            linking_table_path=str(linking_table_old),
+            linking_table_output=str(linking_table_out),
+        )
+
+    assert info.value._message == (
+        "hidden date in [sheet_name='data', 6, 3]"
+        + ' value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"'
+        + " // found=datetime.datetime(2001, 12, 11, 0, 0)"
+    )
+
+
+def test_written(tmp_path: Path) -> None:
+
+    source_file = Path(__file__).parent / "data/hidden_dates/written.xlsx"
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
+
+    sheet_configs = {
+        "yeah": {
+            "patient_id_col": "ptid",
+            "date_columns": [
+                "dob",
+            ],
+            "header_row": 0,
+            "skip_rows_after_header": [],
+        },
+    }
+    with pytest.raises(HiddenDate) as info:
+        shift_excel_dates_inplace(
+            input_file=str(source_file),
+            output_file=str(output_path),
+            patient_sheet="yeah",
+            patient_id_col="ptid",
+            sheet_configs=sheet_configs,
+            min_shift_days=-20,
+            max_shift_days=-1,
+            seed=14333,
+            linking_table_path=str(linking_table_old),
+            linking_table_output=str(linking_table_out),
+        )
+
+    assert info.value._message == (
+        "hidden date in [sheet_name='yeah', 5, 3]"
+        + " value='flu on mar 21st, 2009'"
+        + " // found=datetime.datetime(2009, 3, 21, 0, 0)"
+    )

tests/test_pass_as_is.py

@@ -66,8 +66,8 @@ def body() -> None:
     assert worksheet.cell(3, 3).value == "cheese"
     assert worksheet.cell(4, 3).value == "unknown"
     assert worksheet.cell(5, 3).value == "mushrooms"
-    assert str(worksheet.cell(6, 3).value) == "2016-09-17 00:00:00"
-    assert worksheet.cell(7, 3).value == "2016-07-18 idk"
+    assert worksheet.cell(6, 3).value == "this can't be a date - sorry 2016"
+    assert worksheet.cell(7, 3).value == "idk - this can't be a date anymore"
 
     # the important column to check - the dates
     assert worksheet.cell(1, 2).value == "birthday"