Merge pull request #12 from TheJacksonLaboratory/dev

Data type and path normalization

src/napari_activelearning/_acquisition.py

@@ -255,6 +255,8 @@ def fine_tune(self, dataset_metadata_list: Iterable[
                 shuffle=True,
             )
 
+            dataset.add_transform("images", zds.ToDtype(np.float32))
+
             if USING_PYTORCH:
                 dataloader = DataLoader(
                     dataset,

src/napari_activelearning/_interface.py

@@ -872,8 +872,8 @@ def __init__(self, image_groups_manager: ImageGroupsManagerWidget,
         self.finetuning_btn.clicked.connect(self.fine_tune)
 
         acquisition_lyt = QGridLayout()
-        acquisition_lyt.addWidget(patch_sizes_chk, 0, 0) 
-        acquisition_lyt.addWidget(self.patch_sizes_widget, 1, 0, 1, 3) 
+        acquisition_lyt.addWidget(patch_sizes_chk, 0, 0)
+        acquisition_lyt.addWidget(self.patch_sizes_widget, 1, 0, 1, 3)
         acquisition_lyt.addWidget(QLabel("Maximum samples:"), 2, 0)
         acquisition_lyt.addWidget(self.max_samples_spn, 2, 1)
         acquisition_lyt.addWidget(QLabel("Monte Carlo repetitions"), 3, 0)

src/napari_activelearning/_tests/test_layers.py

@@ -16,6 +16,7 @@
                                            ImageGroupsManager,
                                            ImageGroupEditor,
                                            MaskGenerator)
+from napari_activelearning._utils import get_source_data
 
 
 def test_initialization(single_scale_layer):
@@ -30,7 +31,8 @@ def test_initialization(single_scale_layer):
     assert layer_channel.data_group == data_group
 
     assert ((isinstance(layer_channel.source_data, (Path, str))
-             and layer_channel.source_data == str(input_filename))
+             and (Path(layer_channel.source_data.lower())
+                  == Path(str(input_filename).lower())))
             or np.array_equal(layer_channel.source_data, layer.data))
 
 
@@ -217,13 +219,21 @@ def test_layers_group_properties(single_scale_layer, make_napari_viewer):
     assert layers_group.source_axes == "TCZYX"
 
     expected_metadata = {
-            "modality": "new_sample_layers_group",
-            "filenames": str(input_filename),
-            "data_group": data_group,
-            "source_axes": "TCZYX",
-            "add_to_output": False
-        }
-    assert layers_group.metadata == expected_metadata
+        "modality": "new_sample_layers_group",
+        "filenames": str(input_filename),
+        "data_group": data_group,
+        "source_axes": "TCZYX",
+        "add_to_output": False
+    }
+    assert layers_group.metadata["modality"] == expected_metadata["modality"]
+    assert (Path(layers_group.metadata["filenames"].lower())
+            == Path(expected_metadata["filenames"].lower()))
+    assert (layers_group.metadata["data_group"]
+            == expected_metadata["data_group"])
+    assert (layers_group.metadata["source_axes"]
+            == expected_metadata["source_axes"])
+    assert (layers_group.metadata["add_to_output"]
+            == expected_metadata["add_to_output"])
     viewer.layers.clear()
 
 

src/napari_activelearning/_tests/test_utils.py

@@ -27,13 +27,15 @@ def test_get_source_data(sample_layer):
     input_filename, data_group = get_source_data(layer)
 
     assert (not isinstance(input_filename, (Path, str))
-            or input_filename == str(org_input_filename))
+            or (Path(input_filename.lower())
+                == Path(str(org_input_filename).lower())))
     assert (isinstance(input_filename, (Path, str))
             or (isinstance(input_filename, (MultiScaleData, list))
                 and all(map(np.array_equal, input_filename, org_source_data)))
             or np.array_equal(input_filename, org_source_data))
     assert (not isinstance(input_filename, (Path, str))
-            or data_group == org_data_group)
+            or (Path(str(data_group).lower())
+                == Path(str(org_data_group).lower())))
 
 
 def test_downsample_image(single_scale_type_variant_array):

src/napari_activelearning/_utils.py

@@ -1,6 +1,6 @@
 from typing import Optional, Union, Iterable
-from pathlib import Path
-
+from pathlib import PureWindowsPath, Path
+from urllib.parse import urlparse
 import math
 import tensorstore as ts
 import zarr
@@ -254,6 +254,8 @@ def get_dataloader(
         shuffle=shuffle
     )
 
+    train_dataset.add_transform("images", zds.ToDtype(np.float32))
+
     if USING_PYTORCH:
         train_dataloader = DataLoader(
             train_dataset,
@@ -482,24 +484,54 @@ def get_source_data(layer: Layer):
     data_group = ""
 
     if input_filename:
-        input_filename = Path(input_filename)
-        input_filename_parts = input_filename.parts
+        input_url = urlparse(input_filename)
+
+        input_scheme = input_url.scheme
+        input_netloc = input_url.netloc
+        input_path = Path(input_url.path)
+
+        input_filename_parts = input_path.parts
         extension_idx = list(filter(lambda idx:
                                     ".zarr" in input_filename_parts[idx],
                                     range(len(input_filename_parts))))
         if extension_idx:
             extension_idx = extension_idx[0]
-            data_group = str(Path(*input_filename_parts[extension_idx + 1:]))
-            input_filename = Path(*input_filename_parts[:extension_idx + 1])
+            data_group = Path(
+                *input_filename_parts[extension_idx + 1:]
+            )
+            input_path = Path(
+                *input_filename_parts[:extension_idx + 1]
+            )
+
+        if isinstance(data_group, PureWindowsPath):
+            data_group = data_group.as_posix()
+
+        if isinstance(input_path, PureWindowsPath):
+            input_path = input_path.as_posix()
 
-        input_filename = str(input_filename)
+        input_path = str(input_path)
+
+        if input_scheme:
+            if input_scheme in ["http", "https", "ftp", "s3"]:
+                input_scheme += "://"
+            else:
+                input_scheme += ":"
+
+        input_filename = input_scheme + input_netloc + input_path
+
+        if ".zarr" in input_filename:
+            if data_group:
+                data_group = Path(data_group)
+
+            z_grp = zarr.open(input_filename, mode="r")
+            while not isinstance(z_grp[data_group], zarr.Array):
+                data_group = data_group / "0"
+
+        data_group = str(data_group)
 
     else:
         return layer.data, None
 
-    if input_filename and isinstance(layer.data, (MultiScaleData, list)):
-        data_group = str(Path(data_group) / "0")
-
     if not input_filename:
         input_filename = None