feat: date detection

.gitignore

@@ -12,6 +12,7 @@ wheels/
 
 # Output files
 *.csv
+*.xlsx
 tests/output/
 *.jsonl
 *.jsonl.gz

nuh_helper/date_shift/__init__.py

@@ -20,6 +20,11 @@
 logger = logging.getLogger(__name__)
 
 
+class HiddenDate(Exception):
+    def __init__(
+        self, sheet_name: str, row: int, col: int, value: str, found: datetime
+    ) -> None:
+        message = f"hidden date in [{sheet_name=}, {row}, {col}] {value=} // {found=}"
 class UnknownPatient(Exception):
     def __init__(self, page: str, id: str) -> None:
         message = f"Unknown {id=} on {page=}"
@@ -517,6 +522,19 @@
                 else:
                     cell.value = cast(Any, shifted.to_pydatetime())
 
+            # check for dates in non-date columns
+            for non_date_col_idx in range(1, 1 + (ws.max_column or 0)):
+                if non_date_col_idx in date_col_indices.values():
+                    continue
+
+                value = str(ws.cell(row=row_idx, column=non_date_col_idx).value)
+                import datefinder
+
+                for found in datefinder.find_dates(value):
+                    raise HiddenDate(
+                        sheet_name, row_idx, non_date_col_idx, value, found
+                    )
+
     wb.save(output_file)
     logger.info("Output written to '%s'", output_file)
 

pyproject.toml

@@ -8,6 +8,7 @@ requires-python = ">=3.13"
 dependencies = [
     "pandas>=3.0.5",
     "openpyxl>=3.1.0",
+    "datefinder>=1.0.0",
 ]
 
 [build-system]

tests/data/.gitignore

@@ -0,0 +1,2 @@
+!*.csv
+!*.xlsx

tests/test_hidden_dates.py

@@ -0,0 +1,122 @@
+from pathlib import Path
+
+import pytest
+
+from nuh_helper import shift_excel_dates_inplace
+from nuh_helper.date_shift import (
+    HiddenDate,
+)
+
+
+def test_iso8601(tmp_path: Path) -> None:
+
+    source_file = Path(__file__).parent / "data/hidden_dates/iso8601.xlsx"
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
+
+    sheet_configs = {
+        "args": {
+            "patient_id_col": "ptid",
+            "date_columns": [
+                "dob",
+            ],
+            "header_row": 0,
+            "skip_rows_after_header": [],
+        },
+    }
+    with pytest.raises(HiddenDate) as info:
+        shift_excel_dates_inplace(
+            input_file=str(source_file),
+            output_file=str(output_path),
+            patient_sheet="args",
+            patient_id_col="ptid",
+            sheet_configs=sheet_configs,
+            min_shift_days=-20,
+            max_shift_days=-1,
+            seed=14333,
+            linking_table_path=str(linking_table_old),
+            linking_table_output=str(linking_table_out),
+        )
+
+    assert info.value._message == (
+        "hidden date in [sheet_name='args', 4, 3]"
+        + " value='hives on 2023-10-12'"
+        + " // found=datetime.datetime(2023, 10, 12, 0, 0)"
+    )
+
+
+def test_us_date(tmp_path: Path) -> None:
+
+    source_file = Path(__file__).parent / "data/hidden_dates/us_date.xlsx"
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
+
+    sheet_configs = {
+        "data": {
+            "patient_id_col": "ptid",
+            "date_columns": [
+                "dob",
+            ],
+            "header_row": 0,
+            "skip_rows_after_header": [],
+        },
+    }
+    with pytest.raises(HiddenDate) as info:
+        shift_excel_dates_inplace(
+            input_file=str(source_file),
+            output_file=str(output_path),
+            patient_sheet="data",
+            patient_id_col="ptid",
+            sheet_configs=sheet_configs,
+            min_shift_days=-20,
+            max_shift_days=-1,
+            seed=14333,
+            linking_table_path=str(linking_table_old),
+            linking_table_output=str(linking_table_out),
+        )
+
+    assert info.value._message == (
+        "hidden date in [sheet_name='data', 6, 3]"
+        + ' value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"'
+        + " // found=datetime.datetime(2001, 12, 11, 0, 0)"
+    )
+
+
+def test_written(tmp_path: Path) -> None:
+
+    source_file = Path(__file__).parent / "data/hidden_dates/written.xlsx"
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
+
+    sheet_configs = {
+        "yeah": {
+            "patient_id_col": "ptid",
+            "date_columns": [
+                "dob",
+            ],
+            "header_row": 0,
+            "skip_rows_after_header": [],
+        },
+    }
+    with pytest.raises(HiddenDate) as info:
+        shift_excel_dates_inplace(
+            input_file=str(source_file),
+            output_file=str(output_path),
+            patient_sheet="yeah",
+            patient_id_col="ptid",
+            sheet_configs=sheet_configs,
+            min_shift_days=-20,
+            max_shift_days=-1,
+            seed=14333,
+            linking_table_path=str(linking_table_old),
+            linking_table_output=str(linking_table_out),
+        )
+
+    assert info.value._message == (
+        "hidden date in [sheet_name='yeah', 5, 3]"
+        + " value='flu on mar 21st, 2009'"
+        + " // found=datetime.datetime(2009, 3, 21, 0, 0)"
+    )