fix: supports blank columns

nuh_helper/date_shift/__init__.py

@@ -27,25 +27,19 @@ def __init__(self, page: str, id: str) -> None:
         self._message = message
 
 
-# >> pr 125 Exception goes here
 class ShiftFoundNonDate(Exception):
     def __init__(self, page: str, row: int, col: int, col_name: str, val: str) -> None:
         message = f"{page=}[{row}, {col} @ {col_name=}] {val=}"
         super().__init__(message)
         self._message = message
 
 
-# << end of pr 125
-
 # >> pr 127 Exception goes here
 # << end of pr 127
 
 # >> pr 128 Exception goes here
 # << end of pr 128
 
-# >> pr 129 Exception goes here
-# << end of pr 129
-
 
 class HiddenDate(Exception):
     def __init__(
@@ -85,7 +79,26 @@ def __init__(self, page_name: str, column_name: str) -> None:
         self._column_name = column_name
 
 
+class BlankColumnHasData(Exception):
+    """raised when a column with a blank name has data.
+
+    prevents data being hidden in the wrong part of the CDM"""
+
+    def __init__(self, page: str, row: int, col: int, value: any) -> None:
+        message = f"[{page=} @ {row}, {col}] is a blank column with data {value=}"
+        super().__init__(message)
+        self._message = message
+        self._page: str = page
+        self._row: int = row
+        self._col: int = col
+        self._value: any = value
+
+
 class ExtraColumn(Exception):
+    """raised when an unknown column appears in a page we're shifting.
+
+    could mean a column is named wrong, or, that the sheet_config is incomplete"""
+
     def __init__(self, page_name: str, column_name: str) -> None:
         message = (
             f"{column_name=} is neither ignored or shifted in the cdm {page_name=}"
@@ -542,32 +555,58 @@ def shift_excel_dates_inplace(
         header_values = _excel._get_row_values_resolving_merged(
             ws, header_row_1based, max_col
         )
-        col_index: dict[str, int] = {}
-        for i, val in enumerate(header_values, start=1):
-            if val is not None and str(val).strip():
-                col_index[str(val).strip()] = i
-
-            if (
-                (val not in config["date_columns"])
-                and (val not in config["text_columns"])
-                and (val != config["patient_id_col"])
+        col_indexes: dict[str, int] = {}
+        for col_index, col_name in enumerate(header_values, start=1):
+            # simplify the column name
+            col_name = str(col_name).strip() if col_name is not None else ""
+
+            if col_name == "":
+                # there's no column name - the column should be blank
+                # loop through the values in that column to be sure they're all empty
+                # check each row
+                for row in range(ws.max_row):
+                    row += 1
+
+                    value = ws.cell(row, col_index).value
+
+                    # blank is good
+                    if value is None:
+                        continue
+
+                    # empty strings are also fine
+                    if str(value).strip() == "":
+                        continue
+
+                    # raise an error
+                    raise BlankColumnHasData(sheet_name, row, col_index, value)
+
+                # we don't do more work on blank columns
+
+            elif (
+                # check the config to see if we know what to do with this column
+                (col_name not in config["date_columns"])
+                and (col_name not in config["text_columns"])
+                and (col_name != config["patient_id_col"])
             ):
-                raise ExtraColumn(sheet_name, val)
+                raise ExtraColumn(sheet_name, col_name)
+            else:
+                # happy normal column
+                col_indexes[col_name] = col_index
 
         for text_column in [config["patient_id_col"]] + config["text_columns"]:
             if text_column not in header_values:
                 raise TextColumnMissing(sheet_name, text_column)
 
-        if sheet_patient_id_col not in col_index:
+        if sheet_patient_id_col not in col_indexes:
             raise ValueError(
                 f"Patient ID column '{sheet_patient_id_col}' not found in sheet '{sheet_name}'"  # noqa: E501
             )
 
-        pid_col_idx = col_index[sheet_patient_id_col]
+        pid_col_idx = col_indexes[sheet_patient_id_col]
         date_col_indices: dict[str, int] = {}
         for col in date_columns:
-            if col in col_index:
-                date_col_indices[col] = col_index[col]
+            if col in col_indexes:
+                date_col_indices[col] = col_indexes[col]
             else:
                 raise DateColumnMissing(sheet_name, col)
 

tests/test_structural.py

@@ -4,6 +4,7 @@
 
 from nuh_helper import shift_excel_dates_inplace
 from nuh_helper.date_shift import (
+    BlankColumnHasData,
     DateColumnMissing,
     ExtraColumn,
     ExtraPage,
@@ -12,6 +13,56 @@
 )
 
 
+@pytest.mark.parametrize("good", [True, False])
+def test_with_blank_column(good: bool, tmp_path: Path) -> None:
+
+    source_file = Path(__file__).parent / (
+        "data/structural.with-blank.xlsx" if good else "data/structural.bad-blank.xlsx"
+    )
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
+
+    sheet_configs = {
+        "paige": {
+            "patient_id_col": "ptid",
+            "date_columns": [
+                "dob",
+            ],
+            "text_columns": [
+                "food",
+            ],
+            "header_row": 0,
+            "skip_rows_after_header": [],
+        },
+        "stuff": "skip",
+    }
+
+    def body() -> None:
+        shift_excel_dates_inplace(
+            input_file=str(source_file),
+            output_file=str(output_path),
+            patient_sheet="paige",
+            patient_id_col="ptid",
+            sheet_configs=sheet_configs,
+            min_shift_days=-20,
+            max_shift_days=-1,
+            seed=14333,
+            linking_table_path=str(linking_table_old),
+            linking_table_output=str(linking_table_out),
+        )
+
+    if good:
+        body()
+    else:
+        with pytest.raises(BlankColumnHasData) as info:
+            body()
+        assert info.value._page == "paige"
+        assert info.value._row == 3
+        assert info.value._col == 2
+        assert info.value._value == "forbidden"
+
+
 def test_date_column_missing(tmp_path: Path) -> None:
 
     source_file = Path(__file__).parent / "data/structural.xlsx"