Merge pull request #452 from MannLabs/fix/psm-nan-handling-2_adapt-reader

[fix] Adapt search engine-specific PSM readers to new convention in `_load_file`

Author: lucas-diedrich

alphabase/psm_reader/maxquant_reader.py

@@ -240,13 +240,13 @@ def __init__(  # noqa: PLR0913, D417 # too many arguments in function definition
     def _pre_process(self, df: pd.DataFrame) -> pd.DataFrame:
         """MaxQuant-specific preprocessing of output data."""
         df = df[~pd.isna(df["Retention time"])]
-        df.fillna("", inplace=True)
 
         # remove MBR PSMs as they are currently not supported and will crash import
         mapped_columns = get_column_mapping_for_df(self.column_mapping, df)
         if PsmDfCols.SCAN_NUM in mapped_columns:
             scan_num_col = mapped_columns[PsmDfCols.SCAN_NUM]
-            no_ms2_mask = df[scan_num_col] == ""
+            # MBR PSMs carry no scan number: blank in a text column, NaN in a numeric one
+            no_ms2_mask = df[scan_num_col].isna() | (df[scan_num_col] == "")
             if (num_no_ms2_mask := np.sum(no_ms2_mask)) > 0:
                 warnings.warn(
                     f"MaxQuant PSM file contains {num_no_ms2_mask} MBR PSMs without MS2 scan. This is not yet supported and rows containing MBR PSMs will be removed."

alphabase/psm_reader/msfragger_reader.py

@@ -16,6 +16,7 @@
     psm_reader_provider,
     psm_reader_yaml,
 )
+from alphabase.utils import _sanitize_missing_values
 
 
 def _is_all_fragger_decoy(proteins: list[str]) -> bool:
@@ -415,11 +416,12 @@ def _translate_modifications(self) -> None:
 
     def _load_file(self, filename: str) -> pd.DataFrame:
         """Load MSFragger PSM TSV file."""
-        return pd.read_csv(filename, sep="\t", keep_default_na=False)
+        return _sanitize_missing_values(
+            pd.read_csv(filename, sep="\t", keep_default_na=True)
+        )
 
     def _pre_process(self, df: pd.DataFrame) -> pd.DataFrame:
         """MSFragger PSM TSV preprocessing."""
-        df.fillna("", inplace=True)
         df[[PsmDfCols.RAW_NAME, PsmDfCols.SCAN_NUM]] = (
             df["Spectrum"].str.split(".").apply(lambda x: pd.Series([x[0], int(x[1])]))
         )
@@ -497,11 +499,10 @@ def _translate_modifications(self) -> None:
 
     def _load_file(self, filename: str) -> pd.DataFrame:
         """Load a MsFragger output file to a DataFrame."""
-        return pepxml.DataFrame(filename)
+        return _sanitize_missing_values(pepxml.DataFrame(filename))
 
     def _pre_process(self, df: pd.DataFrame) -> pd.DataFrame:
         """MsFragger-specific preprocessing of output data."""
-        df.fillna("", inplace=True)
         if "ion_mobility" in df.columns:
             df["ion_mobility"] = df["ion_mobility"].astype(float)
         df[PsmDfCols.RAW_NAME] = df["spectrum"].str.split(".").apply(lambda x: x[0])

alphabase/psm_reader/pfind_reader.py

@@ -13,6 +13,7 @@
     PSMReaderBase,
     psm_reader_provider,
 )
+from alphabase.utils import _sanitize_missing_values
 
 
 def _convert_one_pfind_mod(mod: str) -> Optional[str]:  # noqa:  C901 too complex (11 > 10) TODO: refactor
@@ -136,11 +137,12 @@ def _translate_modifications(self) -> None:
 
     def _load_file(self, filename: str) -> pd.DataFrame:
         """Load a pFind output file to a DataFrame."""
-        return pd.read_csv(filename, index_col=False, sep="\t", keep_default_na=False)
+        return _sanitize_missing_values(
+            pd.read_csv(filename, index_col=False, sep="\t", keep_default_na=True)
+        )
 
     def _pre_process(self, df: pd.DataFrame) -> pd.DataFrame:
         """pFind-specific preprocessing of output data."""
-        df.fillna("", inplace=True)
         df = df[df["Sequence"] != ""]
         df[PsmDfCols.RAW_NAME] = df["File_Name"].str.split(".").apply(lambda x: x[0])
         df["Proteins"] = df["Proteins"].apply(parse_pfind_protein)

alphabase/psm_reader/sage_reader.py

@@ -18,6 +18,7 @@
     PSMReaderBase,
     psm_reader_provider,
 )
+from alphabase.utils import _sanitize_missing_values
 
 
 class SageModificationTranslator:
@@ -669,14 +670,14 @@ class SageReaderTSV(SageReaderBase):
     """Reader for Sage output files in TSV format."""
 
     def _load_file(self, filename: str) -> pd.DataFrame:
-        return pd.read_csv(filename, sep="\t")
+        return _sanitize_missing_values(pd.read_csv(filename, sep="\t"))
 
 
 class SageReaderParquet(SageReaderBase):
     """Reader for Sage output files in parquet format."""
 
     def _load_file(self, filename: str) -> pd.DataFrame:
-        return pd.read_parquet(filename)
+        return _sanitize_missing_values(pd.read_parquet(filename))
 
 
 def register_readers() -> None:

alphabase/utils.py

@@ -96,7 +96,7 @@ def _sanitize_missing_values(df: pd.DataFrame) -> pd.DataFrame:
             2    nan
             Name: "object_column", dtype: object
 
-        new_df = _coerce_nan_to_missing_string(df)
+        new_df = _sanitize_missing_values(df)
 
         pd.testing.assert_series_equal(new_df["numeric_column], df["numeric_column"])
         new_df["object_column"]

tests/unit/psm_reader/test_msfragger_psm_tsv.py

@@ -45,11 +45,11 @@ class TestDataProcessing:
     """Tests for data processing and integration."""
 
     def test_preprocessing(self, reader):
-        """Test preprocessing extracts raw names, scan_num, and fills NAs correctly."""
+        """Test preprocessing extracts raw names and scan_num."""
         df = pd.DataFrame(
             {
                 "Spectrum": ["file1.01234.01234.3", "file2.01235.01235.2"],
-                "Peptide": ["PEPTIDE", None],
+                "Peptide": ["PEPTIDE", ""],
                 "Is Decoy": ["false", "true"],
                 "Assigned Modifications": ["5S(79.9663)", ""],
             }