From bd4d63761b3cc6572d047a73f5a138de502a7eeb Mon Sep 17 00:00:00 2001 From: MaximeBICMTL Date: Mon, 22 Dec 2025 17:05:53 +0800 Subject: [PATCH 1/5] Make the EEG chunker into a standalone Python package (#1338) --- .github/labeler.yml | 4 +++ install/templates/environment_template | 2 +- pyproject.toml | 9 +++--- python/lib/physiological.py | 13 +++++---- python/loris_eeg_chunker/README.md | 2 ++ python/loris_eeg_chunker/pyproject.toml | 28 +++++++++++++++++++ .../{ => src/loris_eeg_chunker}/chunking.py | 0 .../protocol_buffers/chunk_pb2.py | 0 .../scripts}/edf_to_chunks.py | 6 +++- .../scripts}/eeglab_to_chunks.py | 6 +++- test/mri.Dockerfile | 2 +- 11 files changed, 57 insertions(+), 15 deletions(-) rename python/loris_eeg_chunker/{ => src/loris_eeg_chunker}/chunking.py (100%) rename python/loris_eeg_chunker/{ => src/loris_eeg_chunker}/protocol_buffers/chunk_pb2.py (100%) rename python/loris_eeg_chunker/{ => src/loris_eeg_chunker/scripts}/edf_to_chunks.py (99%) rename python/loris_eeg_chunker/{ => src/loris_eeg_chunker/scripts}/eeglab_to_chunks.py (99%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 7c88a2069..c2da51c25 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,3 +5,7 @@ "Language: Python": - changed-files: - any-glob-to-any-file: '**/*.py' + +"Package: EEG chunker": +- changed-files: + - any-glob-to-any-file: 'python/loris_eeg_chunker/**' diff --git a/install/templates/environment_template b/install/templates/environment_template index 22fe48d36..403c06a83 100644 --- a/install/templates/environment_template +++ b/install/templates/environment_template @@ -6,7 +6,7 @@ source ${MINC_TOOLKIT_DIR}/minc-toolkit-config.sh umask 0002 # export PATH, PERL5LIB, TMPDIR and LORIS_CONFIG variables -export PATH=/opt/${PROJECT}/bin/mri:/opt/${PROJECT}/bin/mri/uploadNeuroDB:/opt/${PROJECT}/bin/mri/uploadNeuroDB/bin:/opt/${PROJECT}/bin/mri/dicom-archive:/opt/${PROJECT}/bin/mri/python/scripts:/opt/${PROJECT}/bin/mri/tools:/opt/${PROJECT}/bin/mri/python/loris_eeg_chunker:${MINC_TOOLKIT_DIR}/bin:/usr/local/bin/tpcclib:$PATH +export PATH=/opt/${PROJECT}/bin/mri:/opt/${PROJECT}/bin/mri/uploadNeuroDB:/opt/${PROJECT}/bin/mri/uploadNeuroDB/bin:/opt/${PROJECT}/bin/mri/dicom-archive:/opt/${PROJECT}/bin/mri/python/scripts:/opt/${PROJECT}/bin/mri/tools:${MINC_TOOLKIT_DIR}/bin:/usr/local/bin/tpcclib:$PATH export PERL5LIB=/opt/${PROJECT}/bin/mri/uploadNeuroDB:/opt/${PROJECT}/bin/mri/dicom-archive:$PERL5LIB export TMPDIR=/tmp export LORIS_CONFIG=/opt/${PROJECT}/bin/mri/config diff --git a/pyproject.toml b/pyproject.toml index dd4201a94..929bf2b8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,17 +12,13 @@ license-files = ["LICENSE"] requires-python = ">= 3.11" dependencies = [ "boto3==1.35.99", - "google", "mat73", "matplotlib", - "mne", - "mne-bids>=0.14", "mysqlclient", "nibabel", "nilearn", "nose", "numpy", - "protobuf>=3.0.0", "pybids==0.17.0", "pydicom", "python-dateutil", @@ -30,6 +26,7 @@ dependencies = [ "scipy", "sqlalchemy>=2.0.0", "virtualenv", + "loris-eeg-chunker @ {root:uri}/python/loris_eeg_chunker", ] [project.optional-dependencies] @@ -42,10 +39,12 @@ dev = [ [project.urls] Homepage = "https://github.com/aces/loris-mri" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.build.targets.wheel] packages = [ "python/lib", - "python/loris_eeg_chunker", "python/tests", ] diff --git a/python/lib/physiological.py b/python/lib/physiological.py index 97b4f639a..e8dcf7c61 100644 --- a/python/lib/physiological.py +++ b/python/lib/physiological.py @@ -1234,12 +1234,13 @@ def create_chunks_for_visualization(self, physio_file_id, data_dir): # determine which script to run based on the file type file_type = self.grep_file_type_from_file_id(physio_file_id) - if file_type == 'set': - script = os.environ['LORIS_MRI'] + '/python/loris_eeg_chunker/eeglab_to_chunks.py' - command = 'python ' + script + ' ' + full_file_path + ' --destination ' + chunk_root_dir - elif file_type == 'edf': - script = os.environ['LORIS_MRI'] + '/python/loris_eeg_chunker/edf_to_chunks.py' - command = 'python ' + script + ' ' + full_file_path + ' --destination ' + chunk_root_dir + match file_type: + case 'set': + script = 'eeglab-to-chunks' + case 'edf': + script = 'edf-to-chunks' + + command = script + ' ' + full_file_path + ' --destination ' + chunk_root_dir # chunk the electrophysiology dataset if a command was determined above try: diff --git a/python/loris_eeg_chunker/README.md b/python/loris_eeg_chunker/README.md index 4f7046e71..18fd0f892 100644 --- a/python/loris_eeg_chunker/README.md +++ b/python/loris_eeg_chunker/README.md @@ -1,3 +1,5 @@ +# LORIS EEG chunker + Set of scripts to chunk EEG data in smaller bits for the React viewer of LORIS. These scripts were extracted on July 8th, 2019 from the master branch of the diff --git a/python/loris_eeg_chunker/pyproject.toml b/python/loris_eeg_chunker/pyproject.toml index ac31b2e9d..e27726c05 100644 --- a/python/loris_eeg_chunker/pyproject.toml +++ b/python/loris_eeg_chunker/pyproject.toml @@ -1,3 +1,31 @@ +[project] +name = "loris-eeg-chunker" +version = "27.0.0" +description = "The LORIS EEG chunker" +readme = "README.md" +license = "GPL-3.0-or-later" +requires-python = ">= 3.11" +dependencies = [ + "google", + "mne", + "mne-bids>=0.14", + "numpy", + "protobuf>=3.0.0", + "scipy", +] + +[project.scripts] +edf-to-chunks = "loris_eeg_chunker.scripts.edf_to_chunks:main" +eeglab-to-chunks = "loris_eeg_chunker.scripts.eeglab_to_chunks:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/loris_eeg_chunker"] + [tool.ruff] extend = "../../pyproject.toml" +src = ["src"] exclude = ["protocol_buffers"] diff --git a/python/loris_eeg_chunker/chunking.py b/python/loris_eeg_chunker/src/loris_eeg_chunker/chunking.py similarity index 100% rename from python/loris_eeg_chunker/chunking.py rename to python/loris_eeg_chunker/src/loris_eeg_chunker/chunking.py diff --git a/python/loris_eeg_chunker/protocol_buffers/chunk_pb2.py b/python/loris_eeg_chunker/src/loris_eeg_chunker/protocol_buffers/chunk_pb2.py similarity index 100% rename from python/loris_eeg_chunker/protocol_buffers/chunk_pb2.py rename to python/loris_eeg_chunker/src/loris_eeg_chunker/protocol_buffers/chunk_pb2.py diff --git a/python/loris_eeg_chunker/edf_to_chunks.py b/python/loris_eeg_chunker/src/loris_eeg_chunker/scripts/edf_to_chunks.py similarity index 99% rename from python/loris_eeg_chunker/edf_to_chunks.py rename to python/loris_eeg_chunker/src/loris_eeg_chunker/scripts/edf_to_chunks.py index 6434ae07b..23fb60ca9 100755 --- a/python/loris_eeg_chunker/edf_to_chunks.py +++ b/python/loris_eeg_chunker/src/loris_eeg_chunker/scripts/edf_to_chunks.py @@ -13,7 +13,7 @@ def load_channels(exclude): return lambda path : mne.io.read_raw_edf(path, exclude=exclude, preload=False) -if __name__ == '__main__': +def main(): parser = argparse.ArgumentParser( description='Convert .edf files to chunks for browser based visualisation.') parser.add_argument('files', metavar='FILE', type=str, nargs='+', @@ -87,3 +87,7 @@ def load_channels(exclude): destination=args.destination, prefix=args.prefix ) + + +if __name__ == '__main__': + main() diff --git a/python/loris_eeg_chunker/eeglab_to_chunks.py b/python/loris_eeg_chunker/src/loris_eeg_chunker/scripts/eeglab_to_chunks.py similarity index 99% rename from python/loris_eeg_chunker/eeglab_to_chunks.py rename to python/loris_eeg_chunker/src/loris_eeg_chunker/scripts/eeglab_to_chunks.py index 902251a23..070478068 100755 --- a/python/loris_eeg_chunker/eeglab_to_chunks.py +++ b/python/loris_eeg_chunker/src/loris_eeg_chunker/scripts/eeglab_to_chunks.py @@ -13,7 +13,7 @@ def load_channels(path): return mne.io.read_raw_eeglab(path, preload=False) -if __name__ == '__main__': +def main(): parser = argparse.ArgumentParser( description='Convert .set files to chunks for browser based visualisation.') parser.add_argument('files', metavar='FILE', type=str, nargs='+', @@ -57,3 +57,7 @@ def load_channels(path): destination=args.destination, prefix=args.prefix ) + + +if __name__ == '__main__': + main() diff --git a/test/mri.Dockerfile b/test/mri.Dockerfile index 683ce94e7..f5803f16d 100644 --- a/test/mri.Dockerfile +++ b/test/mri.Dockerfile @@ -86,7 +86,7 @@ RUN bash ./test/imaging_install_test.sh $DATABASE_NAME $DATABASE_USER $DATABASE_ # Setup the LORIS-MRI environment variables ENV PROJECT=loris ENV MINC_TOOLKIT_DIR=/opt/minc/1.9.18 -ENV PATH=/opt/${PROJECT}/bin/mri:/opt/${PROJECT}/bin/mri/uploadNeuroDB:/opt/${PROJECT}/bin/mri/uploadNeuroDB/bin:/opt/${PROJECT}/bin/mri/dicom-archive:/opt/${PROJECT}/bin/mri/python/scripts:/opt/${PROJECT}/bin/mri/tools:/opt/${PROJECT}/bin/mri/python/loris_eeg_chunker:${MINC_TOOLKIT_DIR}/bin:/usr/local/bin/tpcclib:$PATH +ENV PATH=/opt/${PROJECT}/bin/mri:/opt/${PROJECT}/bin/mri/uploadNeuroDB:/opt/${PROJECT}/bin/mri/uploadNeuroDB/bin:/opt/${PROJECT}/bin/mri/dicom-archive:/opt/${PROJECT}/bin/mri/python/scripts:/opt/${PROJECT}/bin/mri/tools:${MINC_TOOLKIT_DIR}/bin:/usr/local/bin/tpcclib:$PATH ENV PERL5LIB=/opt/${PROJECT}/bin/mri/uploadNeuroDB:/opt/${PROJECT}/bin/mri/dicom-archive:$PERL5LIB ENV TMPDIR=/tmp ENV LORIS_CONFIG=/opt/${PROJECT}/bin/mri/config From a8aaf76f1d9eab3a8be048a42bf7cfba20eb0b8b Mon Sep 17 00:00:00 2001 From: MaximeBICMTL Date: Mon, 22 Dec 2025 17:07:06 +0800 Subject: [PATCH 2/5] Fix EEG chunks path bug with EEGChunksPath in EEG BIDS import (#1345) --- python/lib/physiological.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/lib/physiological.py b/python/lib/physiological.py index e8dcf7c61..12858defc 100644 --- a/python/lib/physiological.py +++ b/python/lib/physiological.py @@ -1223,12 +1223,13 @@ def create_chunks_for_visualization(self, physio_file_id, data_dir): chunk_root_dir_config = self.config_db_obj.get_config("EEGChunksPath") chunk_root_dir = chunk_root_dir_config - if not chunk_root_dir: - # the bids_rel_dir is the first two directories in file_path ( - # bids_imports/BIDS_dataset_name_BIDSVersion) - file_path_components = Path(file_path).parts - bids_rel_dir = os.path.join(file_path_components[0], file_path_components[1]) - chunk_root_dir = os.path.join(data_dir, f'{bids_rel_dir}_chunks') + file_path_parts = Path(file_path).parts + if chunk_root_dir_config: + chunk_root_dir = chunk_root_dir_config + else: + chunk_root_dir = os.path.join(data_dir, file_path_parts[0]) + + chunk_root_dir = os.path.join(chunk_root_dir, f'{file_path_parts[1]}_chunks') full_file_path = os.path.join(data_dir, file_path) @@ -1261,6 +1262,5 @@ def create_chunks_for_visualization(self, physio_file_id, data_dir): self.insert_physio_parameter_file( physiological_file_id = physio_file_id, parameter_name = 'electrophysiology_chunked_dataset_path', - value = os.path.relpath(chunk_path, chunk_root_dir_config) if chunk_root_dir_config - else os.path.relpath(chunk_path, data_dir) + value = os.path.relpath(chunk_path, data_dir) ) From 9990c2412b6b2eabfa6742198706377ae88117b4 Mon Sep 17 00:00:00 2001 From: MaximeBICMTL Date: Mon, 22 Dec 2025 17:08:12 +0800 Subject: [PATCH 3/5] Add database checks to the EEG BIDS import integration test (#1346) * add eeg bids import database integration test * override raisinbread physiological files * move physio file parameter queries --- python/lib/db/queries/physio_file.py | 15 +++++++ python/lib/db/queries/physio_parameter.py | 21 ++++++++++ python/lib/imaging_lib/physio.py | 15 +++++++ .../scripts/test_import_bids_dataset.py | 41 ++++++++++++++++++- test/RB_SQL/RB_physiological_archive.sql | 0 test/RB_SQL/RB_physiological_channel.sql | 0 ...ysiological_coord_system_electrode_rel.sql | 0 test/RB_SQL/RB_physiological_electrode.sql | 0 .../RB_SQL/RB_physiological_event_archive.sql | 0 test/RB_SQL/RB_physiological_event_file.sql | 0 .../RB_physiological_event_parameter.sql | 0 test/RB_SQL/RB_physiological_file.sql | 0 .../RB_physiological_parameter_file.sql | 0 13 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 python/lib/db/queries/physio_file.py create mode 100644 python/lib/db/queries/physio_parameter.py create mode 100644 python/lib/imaging_lib/physio.py create mode 100644 test/RB_SQL/RB_physiological_archive.sql create mode 100644 test/RB_SQL/RB_physiological_channel.sql create mode 100644 test/RB_SQL/RB_physiological_coord_system_electrode_rel.sql create mode 100644 test/RB_SQL/RB_physiological_electrode.sql create mode 100644 test/RB_SQL/RB_physiological_event_archive.sql create mode 100644 test/RB_SQL/RB_physiological_event_file.sql create mode 100644 test/RB_SQL/RB_physiological_event_parameter.sql create mode 100644 test/RB_SQL/RB_physiological_file.sql create mode 100644 test/RB_SQL/RB_physiological_parameter_file.sql diff --git a/python/lib/db/queries/physio_file.py b/python/lib/db/queries/physio_file.py new file mode 100644 index 000000000..115b32d39 --- /dev/null +++ b/python/lib/db/queries/physio_file.py @@ -0,0 +1,15 @@ +from sqlalchemy import select +from sqlalchemy.orm import Session as Database + +from lib.db.models.physio_file import DbPhysioFile + + +def try_get_physio_file_with_path(db: Database, path: str) -> DbPhysioFile | None: + """ + Get a physiological file from the database using its path, or return `None` if no file was + found. + """ + + return db.execute(select(DbPhysioFile) + .where(DbPhysioFile.path == path) + ).scalar_one_or_none() diff --git a/python/lib/db/queries/physio_parameter.py b/python/lib/db/queries/physio_parameter.py new file mode 100644 index 000000000..b062b47dc --- /dev/null +++ b/python/lib/db/queries/physio_parameter.py @@ -0,0 +1,21 @@ +from collections.abc import Sequence + +from sqlalchemy import select +from sqlalchemy.orm import Session as Database + +from lib.db.models.parameter_type import DbParameterType +from lib.db.models.physio_file_parameter import DbPhysioFileParameter + + +def get_physio_file_parameters( + db: Database, + physio_file_id: int, +) -> Sequence[tuple[DbParameterType, DbPhysioFileParameter]]: + """ + Get the parameters of a physiological file using its ID. + """ + + return db.execute(select(DbParameterType, DbPhysioFileParameter) + .join(DbPhysioFileParameter.type) + .where(DbPhysioFileParameter.file_id == physio_file_id) + ).tuples().all() diff --git a/python/lib/imaging_lib/physio.py b/python/lib/imaging_lib/physio.py new file mode 100644 index 000000000..958cfcd68 --- /dev/null +++ b/python/lib/imaging_lib/physio.py @@ -0,0 +1,15 @@ +from sqlalchemy.orm import Session as Database + +from lib.db.queries.physio_parameter import get_physio_file_parameters + + +def get_physio_file_parameters_dict(db: Database, physio_file_id: int) -> dict[str, str | None]: + """ + Get the parameters of a physiological file as a dictionary mapping from the name of the + parameters to their values. + """ + + parameters = get_physio_file_parameters(db, physio_file_id) + return { + parameter_type.name: parameter.value for parameter_type, parameter in parameters + } diff --git a/python/tests/integration/scripts/test_import_bids_dataset.py b/python/tests/integration/scripts/test_import_bids_dataset.py index 4aff8c657..61af7da47 100644 --- a/python/tests/integration/scripts/test_import_bids_dataset.py +++ b/python/tests/integration/scripts/test_import_bids_dataset.py @@ -1,6 +1,8 @@ from lib.db.queries.candidate import try_get_candidate_with_psc_id from lib.db.queries.config import set_config_with_setting_name +from lib.db.queries.physio_file import try_get_physio_file_with_path from lib.db.queries.session import try_get_session_with_cand_id_visit_label +from lib.imaging_lib.physio import get_physio_file_parameters_dict from tests.util.database import get_integration_database_session from tests.util.file_system import assert_files_exist from tests.util.run_integration_script import run_integration_script @@ -28,7 +30,44 @@ def test_import_eeg_bids_dataset(): session = try_get_session_with_cand_id_visit_label(db, candidate.cand_id, 'V1') assert session is not None - # TODO: Add EEG-specific database checks once the EEG-specific ORM models have been created. + # Check that the physiological file has been inserted in the database. + file = try_get_physio_file_with_path( + db, + 'bids_imports/Face13_BIDSVersion_1.1.0/sub-OTT166/ses-V1/eeg/sub-OTT166_ses-V1_task-faceO_eeg.edf', + ) + assert file is not None + + # Check that the physiological file parameters has been inserted in the database. + file_parameters = get_physio_file_parameters_dict(db, file.id) + assert file_parameters == { + 'TaskName': 'FaceHouseCheck', + 'TaskDescription': 'Visual presentation of oval cropped face and house images both upright and inverted. Rare left or right half oval checkerboards were presetned as targets for keypress response.', # noqa: E501 + 'InstitutionName': 'Brock University', + 'InstitutionAddress': '500 Glenridge Ave, St.Catharines, Ontario', + 'SamplingFrequency': '256', + 'EEGChannelCount': '128', + 'EOGChannelCount': '7', + 'EMGChannelCount': '0', + 'ECGChannelCount': '0', + 'EEGReference': 'CMS', + 'MiscChannelCount': '0', + 'TriggerChannelCount': '0', + 'PowerLineFrequency': '60', + 'EEGPlacementScheme': 'Custom equidistant 128 channel BioSemi montage established in coordination with Judith Schedden McMaster Univertisy', # noqa: E501 + 'Manufacturer': 'BioSemi', + 'CapManufacturer': 'ElectroCap International', + 'HardwareFilters': 'n/a', + 'SoftwareFilters': 'n/a', + 'RecordingType': 'continuous', + 'RecordingDuration': '1119', + 'eegjson_file': 'bids_imports/Face13_BIDSVersion_1.1.0/sub-OTT166/ses-V1/eeg/sub-OTT166_ses-V1_task-faceO_eeg.json', # noqa: E501 + 'physiological_json_file_blake2b_hash': 'f762bbf2e4699fbe47a53f2b7c2f990dc401d7aa57a6b4ba37aa04acdc2748feb42ee058b874a14fae7c14f673280847e40a60771b3c397a0cf5abdb8c05077a', # noqa: E501 + 'physiological_file_blake2b_hash': '8c24c5907b724d659f38c65bfffc754003a586961ea9e25ed5fa0741a2691ed217e29553020230553c84c0b5978edf64624747d31623f6267cbd36eba8b70891', # noqa: E501 + 'electrode_file_blake2b_hash': '0206db2650ae5a07e4225ff87b6fb2a6bfcf6ea088dd8cdfd77d04e9e25a171ffe68878aa156478babb32dcaf9b46459f87fe0516b728278cc0d9372a0d49299', # noqa: E501 + 'channel_file_blake2b_hash': '7b91e3650086ef50ecc00f1c50e17e7ad8dc39c484536bbc2423af4be7d2b50a3a0010f840d457fec68fbfb3e136edf4d616a31bab0ca09ed686f555727341dd', # noqa: E501 + 'event_file_blake2b_hash': '532aa0b52749eb9ee52c2bbb65fa7b1d00d7126cb9a4e10bd4b9dbb4c5527b06e30acdaf17d5806e81d3ce8ad224a9f456e27aba1bf8b92fd43522837c7ffec7', # noqa: E501 + 'electrophysiology_chunked_dataset_path': 'bids_imports/Face13_BIDSVersion_1.1.0_chunks/sub-OTT166_ses-V1_task-faceO_eeg.chunks', # noqa: E501 + } # Check that the BIDS files have been copied. assert_files_exist('/data/loris/bids_imports', { diff --git a/test/RB_SQL/RB_physiological_archive.sql b/test/RB_SQL/RB_physiological_archive.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_channel.sql b/test/RB_SQL/RB_physiological_channel.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_coord_system_electrode_rel.sql b/test/RB_SQL/RB_physiological_coord_system_electrode_rel.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_electrode.sql b/test/RB_SQL/RB_physiological_electrode.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_event_archive.sql b/test/RB_SQL/RB_physiological_event_archive.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_event_file.sql b/test/RB_SQL/RB_physiological_event_file.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_event_parameter.sql b/test/RB_SQL/RB_physiological_event_parameter.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_file.sql b/test/RB_SQL/RB_physiological_file.sql new file mode 100644 index 000000000..e69de29bb diff --git a/test/RB_SQL/RB_physiological_parameter_file.sql b/test/RB_SQL/RB_physiological_parameter_file.sql new file mode 100644 index 000000000..e69de29bb From 025b75e51e066c35dbb482ba77dd884920e5cae1 Mon Sep 17 00:00:00 2001 From: MaximeBICMTL Date: Mon, 22 Dec 2025 17:09:03 +0800 Subject: [PATCH 4/5] Migrate the DICOM study importer to pathlib.Path (path PR 1) (#1349) --- python/lib/config.py | 15 +++--- .../nifti_insertion_pipeline.py | 2 +- .../lib/import_dicom_study/dicom_database.py | 15 +++--- python/lib/import_dicom_study/import_log.py | 13 ++--- python/lib/import_dicom_study/summary_get.py | 9 ++-- .../lib/import_dicom_study/summary_write.py | 9 ++-- python/lib/import_dicom_study/text.py | 6 +-- python/lib/util/crypto.py | 5 +- python/lib/util/fs.py | 12 +++-- python/scripts/import_dicom_study.py | 52 ++++++++++--------- python/scripts/summarize_dicom_study.py | 4 +- 11 files changed, 77 insertions(+), 65 deletions(-) diff --git a/python/lib/config.py b/python/lib/config.py index e011164bc..e503725d2 100644 --- a/python/lib/config.py +++ b/python/lib/config.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from typing import Literal from lib.db.queries.config import try_get_config_with_setting_name @@ -26,15 +27,15 @@ def get_patient_id_dicom_header_config(env: Env) -> Literal['PatientID', 'Patien return patient_id_dicom_header -def get_data_dir_path_config(env: Env) -> str: +def get_data_dir_path_config(env: Env) -> Path: """ Get the LORIS base data directory path from the in-database configuration, or exit the program with an error if that configuration value does not exist or is incorrect. """ - data_dir_path = os.path.normpath(_get_config_value(env, 'dataDirBasepath')) + data_dir_path = Path(_get_config_value(env, 'dataDirBasepath')) - if not os.path.isdir(data_dir_path): + if not data_dir_path.is_dir(): log_error_exit( env, ( @@ -52,20 +53,20 @@ def get_data_dir_path_config(env: Env) -> str: return data_dir_path -def get_dicom_archive_dir_path_config(env: Env) -> str: +def get_dicom_archive_dir_path_config(env: Env) -> Path: """ Get the LORIS DICOM archive directory path from the in-database configuration, or exit the program with an error if that configuration value does not exist or is incorrect. """ - dicom_archive_dir_path = os.path.normpath(_get_config_value(env, 'tarchiveLibraryDir')) + dicom_archive_dir_path = Path(_get_config_value(env, 'tarchiveLibraryDir')) - if not os.path.isdir(dicom_archive_dir_path): + if not dicom_archive_dir_path.is_dir(): log_error_exit( env, ( f"The LORIS DICOM archive directory path configuration value '{dicom_archive_dir_path}' does not refer" - " to an existing diretory." + " to an existing directory." ), ) diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py index 5a241623f..9ec4112f1 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py @@ -689,7 +689,7 @@ def _create_pic_image(self): """ file_info = { 'cand_id': self.session.candidate.cand_id, - 'data_dir_path': self.data_dir, + 'data_dir_path': str(self.data_dir), 'file_rel_path': self.assembly_nifti_rel_path, 'is_4D_dataset': self.json_file_dict['time'] is not None, 'file_id': self.file_id diff --git a/python/lib/import_dicom_study/dicom_database.py b/python/lib/import_dicom_study/dicom_database.py index 90fc9354d..ef2472c44 100644 --- a/python/lib/import_dicom_study/dicom_database.py +++ b/python/lib/import_dicom_study/dicom_database.py @@ -1,5 +1,6 @@ from datetime import datetime from functools import cmp_to_key +from pathlib import Path from sqlalchemy.orm import Session as Database @@ -17,14 +18,14 @@ def insert_dicom_archive( db: Database, dicom_summary: DicomStudySummary, dicom_import_log: DicomStudyImportLog, - archive_location: str, + archive_path: Path, ): """ Insert a DICOM archive in the database. """ dicom_archive = DbDicomArchive() - populate_dicom_archive(dicom_archive, dicom_summary, dicom_import_log, archive_location) + populate_dicom_archive(dicom_archive, dicom_summary, dicom_import_log, archive_path) dicom_archive.date_first_archived = datetime.now() db.add(dicom_archive) db.commit() @@ -37,7 +38,7 @@ def update_dicom_archive( dicom_archive: DbDicomArchive, dicom_summary: DicomStudySummary, dicom_import_log: DicomStudyImportLog, - archive_location: str, + archive_path: Path, ): """ Update a DICOM archive in the database. @@ -47,7 +48,7 @@ def update_dicom_archive( delete_dicom_archive_file_series(db, dicom_archive) # Update the database record with the new DICOM information. - populate_dicom_archive(dicom_archive, dicom_summary, dicom_import_log, archive_location) + populate_dicom_archive(dicom_archive, dicom_summary, dicom_import_log, archive_path) db.commit() # Insert the new DICOM files and series. @@ -58,7 +59,7 @@ def populate_dicom_archive( dicom_archive: DbDicomArchive, dicom_summary: DicomStudySummary, dicom_import_log: DicomStudyImportLog, - archive_location: str, + archive_path: Path, ): """ Populate a DICOM archive database object with information from its DICOM summary and DICOM @@ -83,8 +84,8 @@ def populate_dicom_archive( dicom_archive.creating_user = dicom_import_log.creator_name dicom_archive.sum_type_version = dicom_import_log.summary_version dicom_archive.tar_type_version = dicom_import_log.archive_version - dicom_archive.source_location = dicom_import_log.source_path - dicom_archive.archive_location = archive_location + dicom_archive.source_location = str(dicom_import_log.source_path) + dicom_archive.archive_location = str(archive_path) dicom_archive.scanner_manufacturer = dicom_summary.info.scanner.manufacturer or '' dicom_archive.scanner_model = dicom_summary.info.scanner.model or '' dicom_archive.scanner_serial_number = dicom_summary.info.scanner.serial_number or '' diff --git a/python/lib/import_dicom_study/import_log.py b/python/lib/import_dicom_study/import_log.py index 1edf68a14..3c58e067e 100644 --- a/python/lib/import_dicom_study/import_log.py +++ b/python/lib/import_dicom_study/import_log.py @@ -3,6 +3,7 @@ import socket from dataclasses import dataclass from datetime import datetime +from pathlib import Path from lib.import_dicom_study.text_dict import DictWriter @@ -13,8 +14,8 @@ class DicomStudyImportLog: Information about the past import of a DICOM study. """ - source_path: str - target_path: str + source_path: Path + target_path: Path creator_host: str creator_os: str creator_name: str @@ -32,8 +33,8 @@ def write_dicom_study_import_log_to_string(import_log: DicomStudyImportLog): """ return DictWriter([ - ("Taken from dir", import_log.source_path), - ("Archive target location", import_log.target_path), + ("Taken from dir", str(import_log.source_path)), + ("Archive target location", str(import_log.target_path)), ("Name of creating host", import_log.creator_host), ("Name of host OS", import_log.creator_os), ("Created by user", import_log.creator_name), @@ -46,7 +47,7 @@ def write_dicom_study_import_log_to_string(import_log: DicomStudyImportLog): ]).write() -def write_dicom_study_import_log_to_file(import_log: DicomStudyImportLog, file_path: str): +def write_dicom_study_import_log_to_file(import_log: DicomStudyImportLog, file_path: Path): """ Serialize a DICOM study import log into a text file. """ @@ -56,7 +57,7 @@ def write_dicom_study_import_log_to_file(import_log: DicomStudyImportLog, file_p file.write(string) -def make_dicom_study_import_log(source: str, target: str, tarball_md5_sum: str, zipball_md5_sum: str): +def make_dicom_study_import_log(source: Path, target: Path, tarball_md5_sum: str, zipball_md5_sum: str): """ Create a DICOM study import log from the provided arguments about a DICOM study, as well as the current execution environment. diff --git a/python/lib/import_dicom_study/summary_get.py b/python/lib/import_dicom_study/summary_get.py index 7b6c09053..6ed083f07 100644 --- a/python/lib/import_dicom_study/summary_get.py +++ b/python/lib/import_dicom_study/summary_get.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import pydicom import pydicom.errors @@ -17,7 +18,7 @@ from lib.util.fs import iter_all_dir_files -def get_dicom_study_summary(dicom_study_dir_path: str, verbose: bool): +def get_dicom_study_summary(dicom_study_dir_path: Path, verbose: bool): """ Get information about a DICOM study by reading the files in the DICOM study directory. """ @@ -31,7 +32,7 @@ def get_dicom_study_summary(dicom_study_dir_path: str, verbose: bool): if verbose: print(f"Processing file '{file_rel_path}' ({i}/{len(file_rel_paths)})") - file_path = os.path.join(dicom_study_dir_path, file_rel_path) + file_path = dicom_study_dir_path / file_rel_path try: dicom = pydicom.dcmread(file_path) # type: ignore @@ -112,13 +113,13 @@ def get_dicom_file_info(dicom: pydicom.Dataset) -> DicomStudyDicomFile: ) -def get_other_file_info(file_path: str) -> DicomStudyOtherFile: +def get_other_file_info(file_path: Path) -> DicomStudyOtherFile: """ Get information about a non-DICOM file within a DICOM study. """ return DicomStudyOtherFile( - os.path.basename(file_path), + file_path.name, compute_file_md5_hash(file_path), ) diff --git a/python/lib/import_dicom_study/summary_write.py b/python/lib/import_dicom_study/summary_write.py index aad415ebd..604dc36ac 100644 --- a/python/lib/import_dicom_study/summary_write.py +++ b/python/lib/import_dicom_study/summary_write.py @@ -1,5 +1,6 @@ import xml.etree.ElementTree as ET from functools import cmp_to_key +from pathlib import Path from lib.import_dicom_study.summary_type import ( DicomStudyDicomFile, @@ -14,14 +15,14 @@ from lib.util.iter import count, flatten -def write_dicom_study_summary_to_file(dicom_summary: DicomStudySummary, filename: str): +def write_dicom_study_summary_to_file(dicom_summary: DicomStudySummary, file_path: Path): """ Serialize a DICOM study summary object into a text file. """ - string = write_dicom_study_summary(dicom_summary) - with open(filename, 'w') as file: - file.write(string) + summary = write_dicom_study_summary(dicom_summary) + with open(file_path, 'w') as file: + file.write(summary) def write_dicom_study_summary(dicom_summary: DicomStudySummary) -> str: diff --git a/python/lib/import_dicom_study/text.py b/python/lib/import_dicom_study/text.py index b5f3004fe..01159dd76 100644 --- a/python/lib/import_dicom_study/text.py +++ b/python/lib/import_dicom_study/text.py @@ -3,8 +3,8 @@ different types of values. """ -import os from datetime import date, datetime +from pathlib import Path from lib.util.crypto import compute_file_md5_hash @@ -66,9 +66,9 @@ def read_float_none(string: str | None): return float(string) -def compute_md5_hash_with_name(path: str): +def compute_md5_hash_with_name(path: Path): """ Get the MD5 sum hash of a file with the filename appended. """ - return f'{compute_file_md5_hash(path)} {os.path.basename(path)}' + return f'{compute_file_md5_hash(path)} {path.name}' diff --git a/python/lib/util/crypto.py b/python/lib/util/crypto.py index 72a790512..84c25910c 100644 --- a/python/lib/util/crypto.py +++ b/python/lib/util/crypto.py @@ -1,7 +1,8 @@ import hashlib +from pathlib import Path -def compute_file_blake2b_hash(file_path: str) -> str: +def compute_file_blake2b_hash(file_path: Path | str) -> str: """ Compute the BLAKE2b hash of a file. """ @@ -15,7 +16,7 @@ def compute_file_blake2b_hash(file_path: str) -> str: return hash.hexdigest() -def compute_file_md5_hash(file_path: str) -> str: +def compute_file_md5_hash(file_path: Path | str) -> str: """ Compute the MD5 hash of a file. """ diff --git a/python/lib/util/fs.py b/python/lib/util/fs.py index 126a7c217..411211da5 100644 --- a/python/lib/util/fs.py +++ b/python/lib/util/fs.py @@ -5,6 +5,7 @@ import tempfile from collections.abc import Iterator from datetime import datetime +from pathlib import Path import lib.exitcode from lib.env import Env @@ -26,16 +27,17 @@ def extract_archive(env: Env, tar_path: str, prefix: str, dir_path: str) -> str: return extract_path -def iter_all_dir_files(dir_path: str) -> Iterator[str]: +def iter_all_dir_files(dir_path: Path) -> Iterator[Path]: """ Iterate through all the files in a directory recursively, and yield the path of each file relative to that directory. """ - for sub_dir_path, _, file_names in os.walk(dir_path): - for file_name in file_names: - file_path = os.path.join(sub_dir_path, file_name) - yield os.path.relpath(file_path, start=dir_path) + for item_path in dir_path.iterdir(): + if item_path.is_dir(): + yield from iter_all_dir_files(item_path) + elif item_path.is_file(): + yield item_path.relative_to(dir_path) def remove_directory(env: Env, path: str): diff --git a/python/scripts/import_dicom_study.py b/python/scripts/import_dicom_study.py index bec34dd11..19f549fab 100755 --- a/python/scripts/import_dicom_study.py +++ b/python/scripts/import_dicom_study.py @@ -5,6 +5,7 @@ import shutil import tarfile import tempfile +from pathlib import Path from typing import Any, cast import lib.exitcode @@ -30,7 +31,7 @@ class Args: profile: str - source: str + source: Path insert: bool update: bool session: bool @@ -39,7 +40,7 @@ class Args: def __init__(self, options_dict: dict[str, Any]): self.profile = options_dict['profile']['value'] - self.source = os.path.normpath(options_dict['source']['value']) + self.source = Path(options_dict['source']['value']) self.overwrite = options_dict['overwrite']['value'] self.insert = options_dict['insert']['value'] self.update = options_dict['update']['value'] @@ -108,13 +109,13 @@ def main() -> None: # Get the CLI arguments and connect to the database. - loris_getopt_obj = LorisGetOpt(usage, options_dict, os.path.basename(__file__[:-3])) + loris_getopt_obj = LorisGetOpt(usage, options_dict, 'import_dicom_study') env = make_env(loris_getopt_obj) args = Args(loris_getopt_obj.options_dict) # Check arguments. - if not os.path.isdir(args.source) or not os.access(args.source, os.R_OK): + if not args.source.is_dir() or not os.access(args.source, os.R_OK): log_error_exit( env, "Argument '--source' must be a readable directory path.", @@ -141,7 +142,7 @@ def main() -> None: # Utility variables. - dicom_study_name = os.path.basename(args.source) + dicom_study_name = args.source.name log(env, "Extracting DICOM information... (may take a long time)") @@ -193,24 +194,24 @@ def main() -> None: if dicom_summary.info.scan_date is None: log_warning(env, "No DICOM scan date found in the DICOM files.") - dicom_archive_rel_path = f'DCM_{dicom_study_name}.tar' + dicom_archive_rel_path = Path(f'DCM_{dicom_study_name}.tar') else: log(env, f"Found DICOM scan date: {dicom_summary.info.scan_date}") scan_date_string = lib.import_dicom_study.text.write_date(dicom_summary.info.scan_date) - dicom_archive_rel_path = os.path.join( - str(dicom_summary.info.scan_date.year), - f'DCM_{scan_date_string}_{dicom_study_name}.tar', + dicom_archive_rel_path = ( + Path(str(dicom_summary.info.scan_date.year)) + / f'DCM_{scan_date_string}_{dicom_study_name}.tar' ) - dicom_archive_year_dir_path = os.path.join(dicom_archive_dir_path, str(dicom_summary.info.scan_date.year)) - if not os.path.exists(dicom_archive_year_dir_path): + dicom_archive_year_dir_path = dicom_archive_dir_path / str(dicom_summary.info.scan_date.year) + if not dicom_archive_year_dir_path.exists(): log(env, f"Creating year directory '{dicom_archive_year_dir_path}'...") - os.mkdir(dicom_archive_year_dir_path) + dicom_archive_year_dir_path.mkdir() - dicom_archive_path = os.path.join(dicom_archive_dir_path, dicom_archive_rel_path) + dicom_archive_path = dicom_archive_dir_path / dicom_archive_rel_path - if os.path.exists(dicom_archive_path): + if dicom_archive_path.exists(): if not args.overwrite: log_error_exit( env, @@ -219,20 +220,21 @@ def main() -> None: log_warning(env, f"Overwriting file '{dicom_archive_path}'...") - os.remove(dicom_archive_path) + dicom_archive_path.unlink() - with tempfile.TemporaryDirectory() as tmp_dir_path: - tar_path = os.path.join(tmp_dir_path, f'{dicom_study_name}.tar') - zip_path = os.path.join(tmp_dir_path, f'{dicom_study_name}.tar.gz') - summary_path = os.path.join(tmp_dir_path, f'{dicom_study_name}.meta') - log_path = os.path.join(tmp_dir_path, f'{dicom_study_name}.log') + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir_path = Path(tmp_dir) + tar_path = tmp_dir_path / f'{dicom_study_name}.tar' + zip_path = tmp_dir_path / f'{dicom_study_name}.tar.gz' + summary_path = tmp_dir_path / f'{dicom_study_name}.meta' + log_path = tmp_dir_path / f'{dicom_study_name}.log' log(env, "Copying the DICOM files into a new tar archive...") with tarfile.open(tar_path, 'w') as tar: for file_rel_path in iter_all_dir_files(args.source): - file_path = os.path.join(args.source, file_rel_path) - file_tar_path = os.path.join(os.path.basename(args.source), file_rel_path) + file_path = args.source / file_rel_path + file_tar_path = Path(args.source.name) / file_rel_path tar.add(file_path, arcname=file_tar_path) log(env, "Calculating the tar archive MD5 sum...") @@ -270,9 +272,9 @@ def main() -> None: log(env, 'Copying files into the final DICOM study archive...') with tarfile.open(dicom_archive_path, 'w') as tar: - tar.add(zip_path, os.path.basename(zip_path)) - tar.add(summary_path, os.path.basename(summary_path)) - tar.add(log_path, os.path.basename(log_path)) + tar.add(zip_path, zip_path.name) + tar.add(summary_path, summary_path.name) + tar.add(log_path, log_path.name) log(env, "Calculating final DICOM study archive MD5 sum...") diff --git a/python/scripts/summarize_dicom_study.py b/python/scripts/summarize_dicom_study.py index a0fffbb32..7df1d12cb 100755 --- a/python/scripts/summarize_dicom_study.py +++ b/python/scripts/summarize_dicom_study.py @@ -3,6 +3,7 @@ import argparse import sys from dataclasses import dataclass +from pathlib import Path import lib.exitcode from lib.import_dicom_study.summary_get import get_dicom_study_summary @@ -15,6 +16,7 @@ parser.add_argument( 'directory', + type=Path, help='The DICOM directory') parser.add_argument( @@ -25,7 +27,7 @@ @dataclass class Args: - directory: str + directory: Path verbose: bool From 6ebbfabd0740f0f48356fd78c07f4cdd431c37f2 Mon Sep 17 00:00:00 2001 From: MaximeBICMTL Date: Mon, 22 Dec 2025 17:28:09 +0800 Subject: [PATCH 5/5] Migrate ORM to pathlib.Path (path PR 2) (#1350) --- python/lib/db/decorators/string_path.py | 28 +++++++++++++++++ python/lib/db/models/dicom_archive.py | 6 ++-- python/lib/db/models/file.py | 4 ++- .../db/models/mri_protocol_violated_scan.py | 4 ++- python/lib/db/models/mri_upload.py | 6 ++-- python/lib/db/models/mri_violation_log.py | 4 ++- python/lib/db/models/physio_channel.py | 4 ++- python/lib/db/models/physio_coord_system.py | 15 +++++---- python/lib/db/models/physio_event_file.py | 16 +++++----- python/lib/db/models/physio_file.py | 4 ++- python/lib/db/queries/dicom_archive.py | 8 +++-- python/lib/db/queries/file.py | 10 +++--- python/lib/db/queries/physio_file.py | 4 ++- .../base_pipeline.py | 7 +++-- .../dicom_archive_loader_pipeline.py | 9 +++--- .../dicom_validation_pipeline.py | 2 +- .../nifti_insertion_pipeline.py | 2 +- .../lib/import_dicom_study/dicom_database.py | 4 +-- .../scripts/test_import_bids_dataset.py | 4 ++- .../scripts/test_mass_nifti_pic.py | 3 +- .../scripts/test_run_dicom_archive_loader.py | 6 ++-- .../scripts/test_run_nifti_insertion.py | 31 ++++++++++--------- .../tests/unit/db/query/test_dicom_archive.py | 5 +-- 23 files changed, 124 insertions(+), 62 deletions(-) create mode 100644 python/lib/db/decorators/string_path.py diff --git a/python/lib/db/decorators/string_path.py b/python/lib/db/decorators/string_path.py new file mode 100644 index 000000000..1e7ee5465 --- /dev/null +++ b/python/lib/db/decorators/string_path.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from sqlalchemy import String +from sqlalchemy.engine import Dialect +from sqlalchemy.types import TypeDecorator + + +class StringPath(TypeDecorator[Path]): + """ + Decorator for a database path type. + In SQL, the type will appear as a string. + In Python, the type will appear as a path object. + """ + + impl = String + cache_ok = True + + def process_bind_param(self, value: Path | None, dialect: Dialect) -> str | None: + if value is None: + return None + + return str(value) + + def process_result_value(self, value: str | None, dialect: Dialect) -> Path | None: + if value is None: + return None + + return Path(value) diff --git a/python/lib/db/models/dicom_archive.py b/python/lib/db/models/dicom_archive.py index e5952b321..a13ae0671 100644 --- a/python/lib/db/models/dicom_archive.py +++ b/python/lib/db/models/dicom_archive.py @@ -1,4 +1,5 @@ from datetime import date, datetime +from pathlib import Path from typing import Optional from sqlalchemy import ForeignKey @@ -12,6 +13,7 @@ import lib.db.models.session as db_session from lib.db.base import Base from lib.db.decorators.int_bool import IntBool +from lib.db.decorators.string_path import StringPath class DbDicomArchive(Base): @@ -37,8 +39,8 @@ class DbDicomArchive(Base): creating_user : Mapped[str] = mapped_column('CreatingUser') sum_type_version : Mapped[int] = mapped_column('sumTypeVersion') tar_type_version : Mapped[int | None] = mapped_column('tarTypeVersion') - source_location : Mapped[str] = mapped_column('SourceLocation') - archive_location : Mapped[str | None] = mapped_column('ArchiveLocation') + source_path : Mapped[Path] = mapped_column('SourceLocation', StringPath) + archive_path : Mapped[Path | None] = mapped_column('ArchiveLocation', StringPath) scanner_manufacturer : Mapped[str] = mapped_column('ScannerManufacturer') scanner_model : Mapped[str] = mapped_column('ScannerModel') scanner_serial_number : Mapped[str] = mapped_column('ScannerSerialNumber') diff --git a/python/lib/db/models/file.py b/python/lib/db/models/file.py index 15f2c0b37..21cf10e58 100644 --- a/python/lib/db/models/file.py +++ b/python/lib/db/models/file.py @@ -1,4 +1,5 @@ from datetime import date, datetime +from pathlib import Path from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -8,6 +9,7 @@ from lib.db.base import Base from lib.db.decorators.int_bool import IntBool from lib.db.decorators.int_datetime import IntDatetime +from lib.db.decorators.string_path import StringPath class DbFile(Base): @@ -15,7 +17,7 @@ class DbFile(Base): id : Mapped[int] = mapped_column('FileID', primary_key=True) session_id : Mapped[int] = mapped_column('SessionID', ForeignKey('session.ID')) - rel_path : Mapped[str] = mapped_column('File') + path : Mapped[Path] = mapped_column('File', StringPath) series_uid : Mapped[str | None] = mapped_column('SeriesUID') echo_time : Mapped[float | None] = mapped_column('EchoTime') phase_encoding_direction : Mapped[str | None] = mapped_column('PhaseEncodingDirection') diff --git a/python/lib/db/models/mri_protocol_violated_scan.py b/python/lib/db/models/mri_protocol_violated_scan.py index 8446eca00..7ac1e1e66 100644 --- a/python/lib/db/models/mri_protocol_violated_scan.py +++ b/python/lib/db/models/mri_protocol_violated_scan.py @@ -1,4 +1,5 @@ from datetime import datetime +from pathlib import Path from typing import Optional from sqlalchemy import ForeignKey @@ -8,6 +9,7 @@ import lib.db.models.dicom_archive as db_dicom_archive import lib.db.models.mri_protocol_group as db_mri_protocol_group from lib.db.base import Base +from lib.db.decorators.string_path import StringPath class DbMriProtocolViolatedScan(Base): @@ -19,7 +21,7 @@ class DbMriProtocolViolatedScan(Base): dicom_archive_id : Mapped[int | None] = mapped_column('TarchiveID', ForeignKey('tarchive.TarchiveID')) time_run : Mapped[datetime | None] = mapped_column('time_run') series_description : Mapped[str | None] = mapped_column('series_description') - file_rel_path : Mapped[str | None] = mapped_column('minc_location') + file_path : Mapped[Path | None] = mapped_column('minc_location', StringPath) patient_name : Mapped[str | None] = mapped_column('PatientName') tr_range : Mapped[str | None] = mapped_column('TR_range') te_range : Mapped[str | None] = mapped_column('TE_range') diff --git a/python/lib/db/models/mri_upload.py b/python/lib/db/models/mri_upload.py index bcdf79aec..bfe237670 100644 --- a/python/lib/db/models/mri_upload.py +++ b/python/lib/db/models/mri_upload.py @@ -1,4 +1,5 @@ from datetime import datetime +from pathlib import Path from typing import Optional from sqlalchemy import ForeignKey @@ -8,6 +9,7 @@ import lib.db.models.session as db_session from lib.db.base import Base from lib.db.decorators.int_bool import IntBool +from lib.db.decorators.string_path import StringPath from lib.db.decorators.y_n_bool import YNBool @@ -17,8 +19,8 @@ class DbMriUpload(Base): id : Mapped[int] = mapped_column('UploadID', primary_key=True) uploaded_by : Mapped[str] = mapped_column('UploadedBy') upload_date : Mapped[datetime | None] = mapped_column('UploadDate') - upload_location : Mapped[str] = mapped_column('UploadLocation') - decompressed_location : Mapped[str] = mapped_column('DecompressedLocation') + upload_path : Mapped[Path] = mapped_column('UploadLocation', StringPath) + decompressed_path : Mapped[Path] = mapped_column('DecompressedLocation', StringPath) insertion_complete : Mapped[bool] = mapped_column('InsertionComplete', IntBool) inserting : Mapped[bool | None] = mapped_column('Inserting', IntBool) patient_name : Mapped[str] = mapped_column('PatientName') diff --git a/python/lib/db/models/mri_violation_log.py b/python/lib/db/models/mri_violation_log.py index d97b30695..b31ee9c4e 100644 --- a/python/lib/db/models/mri_violation_log.py +++ b/python/lib/db/models/mri_violation_log.py @@ -1,4 +1,5 @@ from datetime import datetime +from pathlib import Path from typing import Optional from sqlalchemy import ForeignKey @@ -9,6 +10,7 @@ import lib.db.models.mri_protocol_check_group as db_mri_protocol_check_group import lib.db.models.mri_scan_type as db_mri_scan_type from lib.db.base import Base +from lib.db.decorators.string_path import StringPath class DbMriViolationLog(Base): @@ -19,7 +21,7 @@ class DbMriViolationLog(Base): series_uid : Mapped[str | None] = mapped_column('SeriesUID') dicom_archive_id : Mapped[int | None] \ = mapped_column('TarchiveID', ForeignKey('tarchive.TarchiveID')) - file_rel_path : Mapped[str | None] = mapped_column('MincFile') + file_path : Mapped[Path | None] = mapped_column('MincFile', StringPath) patient_name : Mapped[str | None] = mapped_column('PatientName') candidate_id : Mapped[int | None] = mapped_column('CandidateID', ForeignKey('candidate.ID')) visit_label : Mapped[str | None] = mapped_column('Visit_label') diff --git a/python/lib/db/models/physio_channel.py b/python/lib/db/models/physio_channel.py index f3a94a410..a6163262c 100644 --- a/python/lib/db/models/physio_channel.py +++ b/python/lib/db/models/physio_channel.py @@ -1,5 +1,6 @@ from datetime import datetime from decimal import Decimal +from pathlib import Path from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -8,6 +9,7 @@ import lib.db.models.physio_file as db_physio_file import lib.db.models.physio_status_type as db_physio_status_type from lib.db.base import Base +from lib.db.decorators.string_path import StringPath class DbPhysioChannel(Base): @@ -28,7 +30,7 @@ class DbPhysioChannel(Base): reference : Mapped[str | None] = mapped_column('Reference') status_description : Mapped[str | None] = mapped_column('StatusDescription') unit : Mapped[str | None] = mapped_column('Unit') - file_path : Mapped[str | None] = mapped_column('FilePath') + file_path : Mapped[Path | None] = mapped_column('FilePath', StringPath) physio_file : Mapped['db_physio_file.DbPhysioFile'] = relationship('DbPhysioFile', back_populates='channels') channel_type : Mapped['db_physio_channel_type.DbPhysioChannelType'] = relationship('DbPhysioChannelType') diff --git a/python/lib/db/models/physio_coord_system.py b/python/lib/db/models/physio_coord_system.py index 3be11642a..e28beb624 100644 --- a/python/lib/db/models/physio_coord_system.py +++ b/python/lib/db/models/physio_coord_system.py @@ -1,3 +1,5 @@ +from pathlib import Path + from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -6,17 +8,18 @@ import lib.db.models.physio_coord_system_unit as db_physio_coord_system_unit import lib.db.models.physio_modality as db_physio_modality from lib.db.base import Base +from lib.db.decorators.string_path import StringPath class DbPhysioCoordSystem(Base): __tablename__ = 'physiological_coord_system' - id : Mapped[int] = mapped_column('PhysiologicalCoordSystemID', primary_key=True) - name_id : Mapped[int] = mapped_column('NameID', ForeignKey('physiological_coord_system_name.PhysiologicalCoordSystemNameID')) - type_id : Mapped[int] = mapped_column('TypeID', ForeignKey('physiological_coord_system_type.PhysiologicalCoordSystemTypeID')) - unit_id : Mapped[int] = mapped_column('UnitID', ForeignKey('physiological_coord_system_unit.PhysiologicalCoordSystemUnitID')) - modality_id : Mapped[int] = mapped_column('ModalityID', ForeignKey('physiological_modality.PhysiologicalModalityID')) - file_path : Mapped[str | None] = mapped_column('FilePath') + id : Mapped[int] = mapped_column('PhysiologicalCoordSystemID', primary_key=True) + name_id : Mapped[int] = mapped_column('NameID', ForeignKey('physiological_coord_system_name.PhysiologicalCoordSystemNameID')) + type_id : Mapped[int] = mapped_column('TypeID', ForeignKey('physiological_coord_system_type.PhysiologicalCoordSystemTypeID')) + unit_id : Mapped[int] = mapped_column('UnitID', ForeignKey('physiological_coord_system_unit.PhysiologicalCoordSystemUnitID')) + modality_id : Mapped[int] = mapped_column('ModalityID', ForeignKey('physiological_modality.PhysiologicalModalityID')) + file_path : Mapped[Path | None] = mapped_column('FilePath', StringPath) name : Mapped['db_physio_coord_system_name.DbPhysioCoordSystemName'] = relationship('DbPhysioCoordSystemName') type : Mapped['db_physio_coord_system_type.DbPhysioCoordSystemType'] = relationship('DbPhysioCoordSystemType') diff --git a/python/lib/db/models/physio_event_file.py b/python/lib/db/models/physio_event_file.py index 4557144d2..71124f1ee 100644 --- a/python/lib/db/models/physio_event_file.py +++ b/python/lib/db/models/physio_event_file.py @@ -1,4 +1,5 @@ from datetime import datetime +from pathlib import Path from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -9,18 +10,19 @@ import lib.db.models.physio_task_event as db_physio_task_event import lib.db.models.project as db_project from lib.db.base import Base +from lib.db.decorators.string_path import StringPath class DbPhysioEventFile(Base): __tablename__ = 'physiological_event_file' - id : Mapped[int] = mapped_column('EventFileID', primary_key=True) - physio_file_id : Mapped[int | None] = mapped_column('PhysiologicalFileID', ForeignKey('physiological_file.PhysiologicalFileID')) - project_id : Mapped[int | None] = mapped_column('ProjectID', ForeignKey('Project.ProjectID')) - file_type : Mapped[str] = mapped_column('FileType', ForeignKey('ImagingFileTypes.type')) - file_path : Mapped[str | None] = mapped_column('FilePath') - last_update : Mapped[datetime] = mapped_column('LastUpdate') - last_written : Mapped[datetime] = mapped_column('LastWritten') + id : Mapped[int] = mapped_column('EventFileID', primary_key=True) + physio_file_id : Mapped[int | None] = mapped_column('PhysiologicalFileID', ForeignKey('physiological_file.PhysiologicalFileID')) + project_id : Mapped[int | None] = mapped_column('ProjectID', ForeignKey('Project.ProjectID')) + file_type : Mapped[str] = mapped_column('FileType', ForeignKey('ImagingFileTypes.type')) + file_path : Mapped[Path | None] = mapped_column('FilePath', StringPath) + last_update : Mapped[datetime] = mapped_column('LastUpdate') + last_written : Mapped[datetime] = mapped_column('LastWritten') physio_file : Mapped['db_physio_file.DbPhysioFile | None'] = relationship('PhysiologicalFile') project : Mapped['db_project.DbProject | None'] = relationship('Project') diff --git a/python/lib/db/models/physio_file.py b/python/lib/db/models/physio_file.py index a8604a0f5..4c8819ddd 100644 --- a/python/lib/db/models/physio_file.py +++ b/python/lib/db/models/physio_file.py @@ -1,4 +1,5 @@ from datetime import datetime +from pathlib import Path from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -7,6 +8,7 @@ import lib.db.models.physio_modality as db_physio_modality import lib.db.models.physio_output_type as db_physio_output_type from lib.db.base import Base +from lib.db.decorators.string_path import StringPath class DbPhysioFile(Base): @@ -20,7 +22,7 @@ class DbPhysioFile(Base): file_type : Mapped[str | None] = mapped_column('FileType') acquisition_time : Mapped[datetime | None] = mapped_column('AcquisitionTime') inserted_by_user : Mapped[str] = mapped_column('InsertedByUser') - path : Mapped[str] = mapped_column('FilePath') + path : Mapped[Path] = mapped_column('FilePath', StringPath) index : Mapped[int | None] = mapped_column('Index') parent_id : Mapped[int | None] = mapped_column('ParentID') diff --git a/python/lib/db/queries/dicom_archive.py b/python/lib/db/queries/dicom_archive.py index cc8308109..5126ed0de 100644 --- a/python/lib/db/queries/dicom_archive.py +++ b/python/lib/db/queries/dicom_archive.py @@ -1,4 +1,6 @@ +from pathlib import Path + from sqlalchemy import delete, select from sqlalchemy.orm import Session as Database @@ -29,14 +31,14 @@ def try_get_dicom_archive_with_patient_name(db: Database, patient_name: str) -> ).scalar_one_or_none() -def try_get_dicom_archive_with_archive_location(db: Database, archive_location: str) -> DbDicomArchive | None: +def try_get_dicom_archive_with_archive_path(db: Database, archive_path: Path) -> DbDicomArchive | None: """ - Get a DICOM archive from the database using its archive location, or return `None` if no DICOM + Get a DICOM archive from the database using its archive path, or return `None` if no DICOM archive is found. """ return db.execute(select(DbDicomArchive) - .where(DbDicomArchive.archive_location.like(f'%{archive_location}%')) + .where(DbDicomArchive.archive_path.like(f'%{archive_path}%')) ).scalar_one_or_none() diff --git a/python/lib/db/queries/file.py b/python/lib/db/queries/file.py index 395904b2d..a39259069 100644 --- a/python/lib/db/queries/file.py +++ b/python/lib/db/queries/file.py @@ -1,3 +1,5 @@ +from pathlib import Path + from sqlalchemy import delete, select from sqlalchemy.orm import Session as Database @@ -26,14 +28,14 @@ def try_get_file_with_unique_combination( ).scalar_one_or_none() -def try_get_file_with_rel_path(db: Database, rel_path: str) -> DbFile | None: +def try_get_file_with_path(db: Database, path: Path) -> DbFile | None: """ - Get an imaging file from the database using its relative path, or return `None` if no imaging - file is found. + Get an imaging file from the database using its path, or return `None` if no imaging file is + found. """ return db.execute(select(DbFile) - .where(DbFile.rel_path == rel_path) + .where(DbFile.path == path) ).scalar_one_or_none() diff --git a/python/lib/db/queries/physio_file.py b/python/lib/db/queries/physio_file.py index 115b32d39..5b75c1b0c 100644 --- a/python/lib/db/queries/physio_file.py +++ b/python/lib/db/queries/physio_file.py @@ -1,10 +1,12 @@ +from pathlib import Path + from sqlalchemy import select from sqlalchemy.orm import Session as Database from lib.db.models.physio_file import DbPhysioFile -def try_get_physio_file_with_path(db: Database, path: str) -> DbPhysioFile | None: +def try_get_physio_file_with_path(db: Database, path: Path) -> DbPhysioFile | None: """ Get a physiological file from the database using its path, or return `None` if no file was found. diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py index e4351067e..6fe6c8faf 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py @@ -1,11 +1,12 @@ import os import shutil +from pathlib import Path import lib.exitcode from lib.config import get_data_dir_path_config, get_dicom_archive_dir_path_config from lib.database import Database from lib.database_lib.config import Config -from lib.db.queries.dicom_archive import try_get_dicom_archive_with_archive_location +from lib.db.queries.dicom_archive import try_get_dicom_archive_with_archive_path from lib.db.queries.mri_upload import try_get_mri_upload_with_id from lib.get_session_info import SessionConfigError, get_dicom_archive_session_info from lib.imaging import Imaging @@ -149,7 +150,7 @@ def load_mri_upload_and_dicom_archive(self): ) self.dicom_archive = self.mri_upload.dicom_archive - if os.path.join(self.data_dir, 'tarchive', self.dicom_archive.archive_location) != tarchive_path: + if os.path.join(self.data_dir, 'tarchive', self.dicom_archive.archive_path) != tarchive_path: log_error_exit( self.env, f"UploadID {upload_id} and ArchiveLocation {tarchive_path} do not refer to the same upload", @@ -177,7 +178,7 @@ def load_mri_upload_and_dicom_archive(self): elif tarchive_path: archive_location = os.path.relpath(tarchive_path, self.dicom_lib_dir) - dicom_archive = try_get_dicom_archive_with_archive_location(self.env.db, archive_location) + dicom_archive = try_get_dicom_archive_with_archive_path(self.env.db, Path(archive_location)) if dicom_archive is None: log_error_exit( self.env, diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/dicom_archive_loader_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/dicom_archive_loader_pipeline.py index cfd554007..0543e901b 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/dicom_archive_loader_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/dicom_archive_loader_pipeline.py @@ -3,6 +3,7 @@ import re import subprocess import sys +from pathlib import Path import lib.exitcode from lib.dcm2bids_imaging_pipeline_lib.base_pipeline import BasePipeline @@ -35,7 +36,7 @@ def __init__(self, loris_getopt_obj, script_name): self.init_session_info() self.series_uid = self.options_dict["series_uid"]["value"] self.tarchive_path = os.path.join( - self.data_dir, "tarchive", self.dicom_archive.archive_location + self.data_dir, "tarchive", self.dicom_archive.archive_path ) # --------------------------------------------------------------------------------------------- @@ -47,7 +48,7 @@ def __init__(self, loris_getopt_obj, script_name): # Extract DICOM files from the tarchive # --------------------------------------------------------------------------------------------- self.extracted_dicom_dir = self.imaging_obj.extract_files_from_dicom_archive( - os.path.join(self.data_dir, 'tarchive', self.dicom_archive.archive_location), + os.path.join(self.data_dir, 'tarchive', self.dicom_archive.archive_path), self.tmp_dir ) @@ -334,7 +335,7 @@ def _move_and_update_dicom_archive(self): """ acq_date = self.dicom_archive.date_acquired - archive_location = self.dicom_archive.archive_location + archive_location = str(self.dicom_archive.archive_path) pattern = re.compile(r"^[0-9]{4}/") if acq_date and not pattern.match(archive_location): @@ -349,7 +350,7 @@ def _move_and_update_dicom_archive(self): os.replace(self.tarchive_path, new_tarchive_path) self.tarchive_path = new_tarchive_path # add the new archive location to the list of fields to update in the tarchive table - self.dicom_archive.archive_location = new_archive_location + self.dicom_archive.archive_path = Path(new_archive_location) self.dicom_archive.session = self.session diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/dicom_validation_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/dicom_validation_pipeline.py index f4b6abda6..532ff5768 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/dicom_validation_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/dicom_validation_pipeline.py @@ -53,7 +53,7 @@ def _validate_dicom_archive_md5sum(self): log_verbose(self.env, "Verifying DICOM archive md5sum (checksum)") - dicom_archive_path = os.path.join(self.dicom_lib_dir, self.dicom_archive.archive_location) + dicom_archive_path = os.path.join(self.dicom_lib_dir, self.dicom_archive.archive_path) result = _validate_dicom_archive_md5sum(self.env, self.dicom_archive, dicom_archive_path) if not result: # Update the MRI upload. diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py index 9ec4112f1..dbdd3ab49 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py @@ -328,7 +328,7 @@ def _check_if_nifti_file_was_already_inserted(self): error_msg = f"Found a DICOM archive containing DICOM files with the same SeriesUID ({series_uid})" \ f" and EchoTime ({tar_echo_time}) as the one present in the JSON side car file. " \ f" The DICOM archive location containing those DICOM files is " \ - f" {self.dicom_archive.archive_location}. Please, rerun " \ + f" {self.dicom_archive.archive_path}. Please, rerun " \ f" with either --upload_id or --tarchive_path option." # verify that a file with the same MD5 or blake2b hash has not already been inserted diff --git a/python/lib/import_dicom_study/dicom_database.py b/python/lib/import_dicom_study/dicom_database.py index ef2472c44..d2fdc0731 100644 --- a/python/lib/import_dicom_study/dicom_database.py +++ b/python/lib/import_dicom_study/dicom_database.py @@ -84,8 +84,8 @@ def populate_dicom_archive( dicom_archive.creating_user = dicom_import_log.creator_name dicom_archive.sum_type_version = dicom_import_log.summary_version dicom_archive.tar_type_version = dicom_import_log.archive_version - dicom_archive.source_location = str(dicom_import_log.source_path) - dicom_archive.archive_location = str(archive_path) + dicom_archive.source_path = dicom_import_log.source_path + dicom_archive.archive_path = archive_path dicom_archive.scanner_manufacturer = dicom_summary.info.scanner.manufacturer or '' dicom_archive.scanner_model = dicom_summary.info.scanner.model or '' dicom_archive.scanner_serial_number = dicom_summary.info.scanner.serial_number or '' diff --git a/python/tests/integration/scripts/test_import_bids_dataset.py b/python/tests/integration/scripts/test_import_bids_dataset.py index 61af7da47..0bb12b96b 100644 --- a/python/tests/integration/scripts/test_import_bids_dataset.py +++ b/python/tests/integration/scripts/test_import_bids_dataset.py @@ -1,3 +1,5 @@ +from pathlib import Path + from lib.db.queries.candidate import try_get_candidate_with_psc_id from lib.db.queries.config import set_config_with_setting_name from lib.db.queries.physio_file import try_get_physio_file_with_path @@ -33,7 +35,7 @@ def test_import_eeg_bids_dataset(): # Check that the physiological file has been inserted in the database. file = try_get_physio_file_with_path( db, - 'bids_imports/Face13_BIDSVersion_1.1.0/sub-OTT166/ses-V1/eeg/sub-OTT166_ses-V1_task-faceO_eeg.edf', + Path('bids_imports/Face13_BIDSVersion_1.1.0/sub-OTT166/ses-V1/eeg/sub-OTT166_ses-V1_task-faceO_eeg.edf'), ) assert file is not None diff --git a/python/tests/integration/scripts/test_mass_nifti_pic.py b/python/tests/integration/scripts/test_mass_nifti_pic.py index d21660ee0..db822d35c 100644 --- a/python/tests/integration/scripts/test_mass_nifti_pic.py +++ b/python/tests/integration/scripts/test_mass_nifti_pic.py @@ -1,5 +1,6 @@ import os from datetime import datetime +from pathlib import Path from lib.db.models.file import DbFile from lib.db.queries.file import delete_file @@ -163,7 +164,7 @@ def test_running_on_a_text_file(): # insert fake text file file = DbFile( - rel_path = 'test.txt', + path = Path('test.txt'), file_type = 'txt', session_id = 564, output_type = 'native', diff --git a/python/tests/integration/scripts/test_run_dicom_archive_loader.py b/python/tests/integration/scripts/test_run_dicom_archive_loader.py index 041a5fee8..a2477dd70 100644 --- a/python/tests/integration/scripts/test_run_dicom_archive_loader.py +++ b/python/tests/integration/scripts/test_run_dicom_archive_loader.py @@ -1,3 +1,5 @@ +from pathlib import Path + from lib.db.queries.config import set_config_with_setting_name from lib.db.queries.mri_upload import get_mri_upload_with_patient_name from lib.exitcode import GETOPT_FAILURE, INVALID_PATH, SELECT_FAILURE, SUCCESS @@ -85,7 +87,7 @@ def test_successful_run_on_valid_tarchive_path(): }) # Check that the expected data has been inserted in the database - archive_new_path = '2015/DCM_2015-07-07_MTL001_300001_V2_localizer_t1w.tar' + archive_new_path = Path('2015/DCM_2015-07-07_MTL001_300001_V2_localizer_t1w.tar') mri_upload = get_mri_upload_with_patient_name(db, 'MTL001_300001_V2') # check mri_upload flags assert mri_upload.inserting is False @@ -97,7 +99,7 @@ def test_successful_run_on_valid_tarchive_path(): assert mri_upload.dicom_archive is not None assert mri_upload.dicom_archive.session is not None # check that archive location has been updated - assert mri_upload.dicom_archive.archive_location == archive_new_path + assert mri_upload.dicom_archive.archive_path == archive_new_path # check series/files counts # notes: - tarchive_series should have 2 series for this upload (localizer + T1W) # - localizer is skipped from conversion because of config settings `excluded_series_description` diff --git a/python/tests/integration/scripts/test_run_nifti_insertion.py b/python/tests/integration/scripts/test_run_nifti_insertion.py index 46d9f06f3..4746ccd87 100644 --- a/python/tests/integration/scripts/test_run_nifti_insertion.py +++ b/python/tests/integration/scripts/test_run_nifti_insertion.py @@ -1,6 +1,7 @@ import os.path import shutil from os.path import basename +from pathlib import Path from lib.db.queries.file import try_get_file_with_unique_combination from lib.db.queries.file_parameter import try_get_parameter_value_with_file_id_parameter_name @@ -320,12 +321,12 @@ def test_nifti_mri_protocol_violated_scans_features(): violated_scan_entry = violated_scans[0] # Check that the NIfTI file can be found on the disk - assert violated_scan_entry.file_rel_path is not None - assert os.path.exists(os.path.join('/data/loris/', violated_scan_entry.file_rel_path)) + assert violated_scan_entry.file_path is not None + assert os.path.exists(os.path.join('/data/loris/', violated_scan_entry.file_path)) # Rerun the script to test that it did not duplicate the entry in MRI protocol violated scans # Note: need to copy the violated file into incoming to rerun the script - new_nifti_path = os.path.join('/data/loris/', violated_scan_entry.file_rel_path) + new_nifti_path = os.path.join('/data/loris/', violated_scan_entry.file_path) new_json_path = new_nifti_path.replace('.nii.gz', '.json') shutil.copyfile(new_nifti_path, nifti_path) shutil.copyfile(new_json_path, json_path) @@ -382,7 +383,7 @@ def test_nifti_mri_protocol_violated_scans_features(): # Check that all files related to that image have been properly linked in the database file_base_rel_path = 'assembly_bids/sub-400184/ses-V3/anat/sub-400184_ses-V3_run-1_T1w' - assert file.rel_path == f'{file_base_rel_path}.nii.gz' + assert file.path == Path(f'{file_base_rel_path}.nii.gz') file_json_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'bids_json_file') file_pic_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'check_pic_filename') assert file_json_data is not None and file_json_data.value == f'{file_base_rel_path}.json' @@ -393,7 +394,7 @@ def test_nifti_mri_protocol_violated_scans_features(): 'sub-400184': { 'ses-V3': { 'anat': { - basename(file.rel_path): None, + basename(file.path): None, basename(str(file_json_data.value)): None, } } @@ -463,13 +464,13 @@ def test_nifti_mri_violations_log_exclude_features(): # can be found on the disk assert len(violations) == 1 violation_entry = violations[0] - assert violation_entry.file_rel_path is not None + assert violation_entry.file_path is not None assert violation_entry.severity == 'exclude' # Check that the NIfTI file can be found in the filesystem - assert os.path.exists(os.path.join('/data/loris/', violation_entry.file_rel_path)) + assert os.path.exists(os.path.join('/data/loris/', violation_entry.file_path)) # Check that the rest of the expected files have been created - new_nifti_path = os.path.join('/data/loris/', violation_entry.file_rel_path) + new_nifti_path = os.path.join('/data/loris/', violation_entry.file_path) new_json_path = new_nifti_path.replace('.nii.gz', '.json') new_bval_path = new_nifti_path.replace('.nii.gz', '.bval') new_bvec_path = new_nifti_path.replace('.nii.gz', '.bvec') @@ -539,7 +540,7 @@ def test_nifti_mri_violations_log_exclude_features(): # Check that all files related to that image have been properly linked in the database file_base_rel_path = 'assembly_bids/sub-400184/ses-V3/dwi/sub-400184_ses-V3_acq-65dir_run-1_dwi' - assert file.rel_path == f'{file_base_rel_path}.nii.gz' + assert file.path == Path(f'{file_base_rel_path}.nii.gz') file_json_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'bids_json_file') file_bval_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'check_bval_filename') file_bvec_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'check_bvec_filename') @@ -554,7 +555,7 @@ def test_nifti_mri_violations_log_exclude_features(): 'sub-400184': { 'ses-V3': { 'dwi': { - basename(file.rel_path): None, + basename(file.path): None, basename(str(file_bval_data.value)): None, basename(str(file_bvec_data.value)): None, basename(str(file_json_data.value)): None, @@ -617,14 +618,14 @@ def test_dwi_insertion_with_mri_violations_log_warning(): assert file is not None assert len(violations) == 1 violation_entry = violations[0] - assert violation_entry.file_rel_path is not None + assert violation_entry.file_path is not None assert violation_entry.severity == 'warning' # Check that all files related to that image have been properly linked in the database file_base_rel_path = 'assembly_bids/sub-400184/ses-V3/dwi/sub-400184_ses-V3_acq-25dir_run-1_dwi' - assert violation_entry.file_rel_path \ - == file.rel_path \ - == f'{file_base_rel_path}.nii.gz' + assert violation_entry.file_path \ + == file.path \ + == Path(f'{file_base_rel_path}.nii.gz') file_json_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'bids_json_file') file_bval_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'check_bval_filename') file_bvec_data = try_get_parameter_value_with_file_id_parameter_name(db, file.id, 'check_bvec_filename') @@ -639,7 +640,7 @@ def test_dwi_insertion_with_mri_violations_log_warning(): 'sub-400184': { 'ses-V3': { 'dwi': { - basename(file.rel_path): None, + basename(file.path): None, basename(str(file_bval_data.value)): None, basename(str(file_bvec_data.value)): None, basename(str(file_json_data.value)): None, diff --git a/python/tests/unit/db/query/test_dicom_archive.py b/python/tests/unit/db/query/test_dicom_archive.py index 102790257..a06361a76 100644 --- a/python/tests/unit/db/query/test_dicom_archive.py +++ b/python/tests/unit/db/query/test_dicom_archive.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from pathlib import Path import pytest from sqlalchemy import select @@ -33,7 +34,7 @@ def setup(): creating_user = 'admin', sum_type_version = 2, tar_type_version = 2, - source_location = '/tests/DCC001_111111_V1', + source_path = Path('/tests/DCC001_111111_V1'), scanner_manufacturer = 'Test scanner manufacturer', scanner_model = 'Test scanner model', scanner_serial_number = 'Test scanner serial number', @@ -54,7 +55,7 @@ def setup(): creating_user = 'admin', sum_type_version = 2, tar_type_version = 2, - source_location = '/test/DCC002_222222_V2', + source_path = Path('/test/DCC002_222222_V2'), scanner_manufacturer = 'Test scanner manufacturer', scanner_model = 'Test scanner model', scanner_serial_number = 'Test scanner serial number',