From 82aff0634fb9d8b328d545679caeaaaa0335415e Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Wed, 8 Jan 2025 13:57:34 -0500 Subject: [PATCH 01/10] Added builder for kaggle --- src/portal_visualization/builder_factory.py | 10 +- .../builders/imaging_builders.py | 203 ++++++++++-------- src/vis-preview.py | 24 ++- 3 files changed, 140 insertions(+), 97 deletions(-) diff --git a/src/portal_visualization/builder_factory.py b/src/portal_visualization/builder_factory.py index 9e6c61b..93c43c8 100644 --- a/src/portal_visualization/builder_factory.py +++ b/src/portal_visualization/builder_factory.py @@ -9,6 +9,7 @@ IMSViewConfBuilder, ImagePyramidViewConfBuilder, SegImagePyramidViewConfBuilder, + KaggleSegImagePyramidViewConfBuilder, NanoDESIViewConfBuilder, ) from .builders.anndata_builders import ( @@ -34,6 +35,7 @@ def process_hints(hints): is_json = "json_based" in hints is_spatial = "spatial" in hints is_support = "is_support" in hints + is_seg_mask = "segmentation_mask" in hints return ( is_image, @@ -45,6 +47,7 @@ def process_hints(hints): is_json, is_spatial, is_support, + is_seg_mask, ) @@ -66,14 +69,17 @@ def get_view_config_builder(entity, get_entity, parent=None, epic_uuid=None): is_anndata, is_json, is_spatial, - is_support + is_support, + is_seg_mask, ) = process_hints(hints) # vis-lifted image pyramids if parent is not None: # TODO: For now epic (base image's) support datasets doesn't have any hints - if epic_uuid is not None: + if is_seg_mask and epic_uuid: return SegImagePyramidViewConfBuilder + elif is_seg_mask: + return KaggleSegImagePyramidViewConfBuilder elif is_support and is_image: ancestor_assaytype = get_entity(parent).get("soft_assaytype") diff --git a/src/portal_visualization/builders/imaging_builders.py b/src/portal_visualization/builders/imaging_builders.py index d125fea..0ce2161 100644 --- a/src/portal_visualization/builders/imaging_builders.py +++ b/src/portal_visualization/builders/imaging_builders.py @@ -5,6 +5,9 @@ VitessceConfig, MultiImageWrapper, OmeTiffWrapper, + CoordinationLevel as CL, + get_initial_coordination_scope_prefix, + ObsSegmentationsOmeTiffWrapper, ImageOmeTiffWrapper, Component as cm, ) @@ -17,6 +20,12 @@ class AbstractImagingViewConfBuilder(ViewConfBuilder): + def __init__(self, entity, groups_token, assets_endpoint, **kwargs): + self.image_pyramid_regex = None + self.use_full_resolution = [] + self.use_physical_size_scaling = False + super().__init__(entity, groups_token, assets_endpoint, **kwargs) + def _get_img_and_offset_url(self, img_path, img_dir): """Create a url for the offsets and img. :param str img_path: The path of the image @@ -58,129 +67,134 @@ def _get_img_and_offset_url_seg(self, img_path, img_dir): """ img_url = self._build_assets_url(img_path) - offset_path = f'{SEGMENTATION_SUBDIR}/{OFFSETS_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}' + offsets_path= re.sub(IMAGE_PYRAMID_DIR, OFFSETS_DIR, img_dir) return ( img_url, str( re.sub( r"ome\.tiff?", "offsets.json", - re.sub(img_dir, offset_path, img_url), + re.sub(img_dir, offsets_path, img_url), ) ), ) - def _setup_view_config_raster(self, vc, dataset, disable_3d=[], use_full_resolution=[]): - vc.add_view(cm.SPATIAL, dataset=dataset, x=3, y=0, w=9, h=12).set_props( - useFullResolutionImage=use_full_resolution - ) - vc.add_view(cm.DESCRIPTION, dataset=dataset, x=0, y=8, w=3, h=4) - vc.add_view(cm.LAYER_CONTROLLER, dataset=dataset, x=0, y=0, w=3, h=8).set_props( - disable3d=disable_3d, disableChannelsIfRgbDetected=True - ) - return vc + def _setup_view_config(self, vc, dataset, view_type, disable_3d=[], use_full_resolution=[]): + if view_type == "raster": + vc.add_view(cm.SPATIAL, dataset=dataset, x=3, y=0, w=9, h=12).set_props( + useFullResolutionImage=use_full_resolution + ) + vc.add_view(cm.DESCRIPTION, dataset=dataset, x=0, y=8, w=3, h=4) + vc.add_view(cm.LAYER_CONTROLLER, dataset=dataset, x=0, y=0, w=3, h=8).set_props( + disable3d=disable_3d, disableChannelsIfRgbDetected=True + ) + elif "seg" in view_type: + spatial_view = vc.add_view("spatialBeta", dataset=dataset, x=4, y=0, w=8, h=12).set_props( + useFullResolutionImage=use_full_resolution + ) + lc_view = vc.add_view("layerControllerBeta", dataset=dataset, x=0, y=0, w=4, h=8).set_props( + disable3d=disable_3d, disableChannelsIfRgbDetected=True + ) + # Adding the segmentation mask on top of the image + if view_type == 'kaggle_seg': + vc.link_views_by_dict([spatial_view, lc_view], { + # Neutralizing the base-image colors + 'imageLayer': CL([{'photometricInterpretation': 'RGB', }]), + }, meta=True, scope_prefix=get_initial_coordination_scope_prefix("A", "image")) - def _setup_view_config_seg(self, vc, dataset, disable_3d=[], use_full_resolution=[]): - vc.add_view("spatialBeta", dataset=dataset, x=4, y=0, w=8, h=12).set_props( - useFullResolutionImage=use_full_resolution - ) - vc.add_view("layerControllerBeta", dataset=dataset, x=0, y=0, w=4, h=8).set_props( - disable3d=disable_3d, disableChannelsIfRgbDetected=True - ) return vc - -class ImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): - def __init__(self, entity, groups_token, assets_endpoint, **kwargs): - """Wrapper class for creating a standard view configuration for image pyramids, - i.e for high resolution viz-lifted imaging datasets like - https://portal.hubmapconsortium.org/browse/dataset/dc289471333309925e46ceb9bafafaf4 - """ - self.image_pyramid_regex = IMAGE_PYRAMID_DIR - self.use_full_resolution = [] - self.use_physical_size_scaling = False - super().__init__(entity, groups_token, assets_endpoint, **kwargs) - - def get_conf_cells(self, **kwargs): + def get_conf_cells_common(self, get_img_and_offset_url_func, **kwargs): file_paths_found = self._get_file_paths() - found_images = [ - path for path in get_matches( - file_paths_found, self.image_pyramid_regex + r".*\.ome\.tiff?$", - ) - if 'separate/' not in path # Exclude separate/* in MALDI-IMS - ] + found_images = get_found_images(self.image_pyramid_regex, file_paths_found) found_images = sorted(found_images) - if len(found_images) == 0: + if len(found_images) == 0: # pragma: no cover message = f"Image pyramid assay with uuid {self._uuid} has no matching files" raise FileNotFoundError(message) vc = VitessceConfig(name="HuBMAP Data Portal", schema_version=self._schema_version) dataset = vc.add_dataset(name="Visualization Files") - images = [] - for img_path in found_images: - img_url, offsets_url = self._get_img_and_offset_url( - img_path, self.image_pyramid_regex + + if 'seg' in self.view_type: + img_url, offsets_url = get_img_and_offset_url_func(found_images[0], self.image_pyramid_regex) + dataset = dataset.add_object( + ImageOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, name=Path(found_images[0]).name) ) - images.append( - OmeTiffWrapper( - img_url=img_url, offsets_url=offsets_url, name=Path(img_path).name + if self.view_type == 'kaggle-seg': + file_paths_found = self._get_file_paths() + found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) + filtered_images = [img_path for img_path in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img_path] + if len(filtered_images) == 0: # pragma: no cover + message = f"Image pyramid assay with uuid {self._uuid} has no matching files" + raise FileNotFoundError(message) + + elif len(filtered_images) >= 1: + img_url, offsets_url = self._get_img_and_offset_url( + filtered_images[0], self.seg_image_pyramid_regex + ) + dataset = dataset.add_object( + ObsSegmentationsOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, obs_types_from_channel_names=True)) + + else: + images = [] + for img_path in found_images: + img_url, offsets_url = get_img_and_offset_url_func( + img_path, self.image_pyramid_regex + ) + images.append( + OmeTiffWrapper( + img_url=img_url, offsets_url=offsets_url, name=Path(img_path).name + ) + ) + dataset = dataset.add_object( + MultiImageWrapper( + images, + use_physical_size_scaling=self.use_physical_size_scaling ) ) - dataset = dataset.add_object( - MultiImageWrapper( - images, - use_physical_size_scaling=self.use_physical_size_scaling - ) - ) - vc = self._setup_view_config_raster( - vc, dataset, use_full_resolution=self.use_full_resolution) - conf = vc.to_dict() - # Don't want to render all layers - del conf["datasets"][0]["files"][0]["options"]["renderLayers"] + conf = self._setup_view_config(vc, dataset, self.view_type, use_full_resolution=self.use_full_resolution).to_dict() + if "raster" in self.view_type: + del conf["datasets"][0]["files"][0]["options"]["renderLayers"] return get_conf_cells(conf) -class SegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): - def __init__(self, entity, groups_token, assets_endpoint, **kwargs): - """Wrapper class for creating a standard view configuration for image pyramids for segmenation mask, +class ImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): + """Wrapper class for creating a standard view configuration for image pyramids, i.e for high resolution viz-lifted imaging datasets like - https://portal.hubmapconsortium.org/browse/dataset/ - """ - self.image_pyramid_regex = f'{SEGMENTATION_SUBDIR}/{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}' - self.use_full_resolution = [] - self.use_physical_size_scaling = False + https://portal.hubmapconsortium.org/browse/dataset/dc289471333309925e46ceb9bafafaf4 + """ + def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) + self.image_pyramid_regex = IMAGE_PYRAMID_DIR + self.view_type = "raster" def get_conf_cells(self, **kwargs): - file_paths_found = self._get_file_paths() - found_images = [ - path for path in get_matches( - file_paths_found, self.image_pyramid_regex + r".*\.ome\.tiff?$", - ) - if 'separate/' not in path # Exclude separate/* in MALDI-IMS - ] - found_images = sorted(found_images) - if len(found_images) == 0: # pragma: no cover - message = f"Image pyramid assay with uuid {self._uuid} has no matching files" - raise FileNotFoundError(message) + return self.get_conf_cells_common(self._get_img_and_offset_url, **kwargs) - vc = VitessceConfig(name="HuBMAP Data Portal", schema_version=self._schema_version) - dataset = vc.add_dataset(name="Visualization Files") - # The base-image will always be 1 - if len(found_images) == 1: - img_url, offsets_url = self._get_img_and_offset_url_seg( - found_images[0], self.image_pyramid_regex - ) - image = ImageOmeTiffWrapper( - img_url=img_url, offsets_url=offsets_url, name=Path(found_images[0]).name - ) +class SegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): + """Wrapper class for creating a standard view configuration for image pyramids for segmenation mask, + i.e for high resolution viz-lifted imaging datasets like + https://portal.hubmapconsortium.org/browse/dataset/ + """ + def __init__(self, entity, groups_token, assets_endpoint, **kwargs): + super().__init__(entity, groups_token, assets_endpoint, **kwargs) + self.image_pyramid_regex = f"{SEGMENTATION_SUBDIR}/{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}" + self.view_type = "seg" - dataset = dataset.add_object(image) - vc = self._setup_view_config_seg( - vc, dataset, use_full_resolution=self.use_full_resolution) - conf = vc.to_dict() - return get_conf_cells(conf) + def get_conf_cells(self, **kwargs): + return self.get_conf_cells_common(self._get_img_and_offset_url_seg, **kwargs) + +class KaggleSegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): + # The difference from EPIC segmentation is only the file path and transformations + def __init__(self, entity, groups_token, assets_endpoint, **kwargs): + super().__init__(entity, groups_token, assets_endpoint, **kwargs) + self.image_pyramid_regex = f"{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}" + self.seg_image_pyramid_regex = IMAGE_PYRAMID_DIR + self.view_type = "kaggle-seg" + + def get_conf_cells(self, **kwargs): + return self.get_conf_cells_common(self._get_img_and_offset_url_seg, **kwargs) class IMSViewConfBuilder(ImagePyramidViewConfBuilder): @@ -266,3 +280,12 @@ def _get_pos_name(self, image_path): return re.search(SEQFISH_FILE_REGEX, image_path)[0].split(".")[ 0 ] + +def get_found_images(image_pyramid_regex, file_paths_found): + found_images = [ + path for path in get_matches( + file_paths_found, image_pyramid_regex + r".*\.ome\.tiff?$", + ) + if 'separate/' not in path + ] + return found_images \ No newline at end of file diff --git a/src/vis-preview.py b/src/vis-preview.py index 86ca999..b528641 100755 --- a/src/vis-preview.py +++ b/src/vis-preview.py @@ -11,10 +11,9 @@ from portal_visualization.builder_factory import get_view_config_builder from portal_visualization.epic_factory import get_epic_builder - +defaults = json.load((Path(__file__).parent / 'defaults.json').open()) def main(): # pragma: no cover - defaults = json.load((Path(__file__).parent / 'defaults.json').open()) assets_default_url = defaults['assets_url'] parser = argparse.ArgumentParser(description=''' @@ -53,7 +52,7 @@ def main(): # pragma: no cover parent_uuid = args.parent_uuid headers = get_headers(args.token) - entity = get_entity(args.url, args.json, headers) + entity = get_entity_from_args(args.url, args.json, headers) Builder = get_view_config_builder(entity, get_entity, parent_uuid, epic_uuid) builder = Builder(entity, args.token, args.assets_url) @@ -87,13 +86,28 @@ def main(): # pragma: no cover def get_headers(token): # pragma: no cover + global headers headers = {} if token: headers['Authorization'] = f'Bearer {token}' return headers - -def get_entity(url_arg, json_arg, headers): # pragma: no cover +def get_entity(uuid): + try: + response = requests.get(f'{defaults["dataset_url"]}{uuid}.json', headers=headers) + if response.status_code != 200: + print(f"Error: Received status code {response.status_code}") + else: + try: + data = response.json() + return data + except Exception as e: + print(f"Error in parsing the response {str(e)}") + except Exception as e: + print(f"Error accessing {defaults['assaytypes_url']}{uuid}: {str(e)}") + + +def get_entity_from_args(url_arg, json_arg, headers): # pragma: no cover if url_arg: response = requests.get(url_arg, headers=headers) if response.status_code == 403: From f0205c3f4a8baeb361ad52fd3db8bcb6e51b7e76 Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Wed, 8 Jan 2025 16:17:09 -0500 Subject: [PATCH 02/10] Refactored code --- .../builders/imaging_builders.py | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/src/portal_visualization/builders/imaging_builders.py b/src/portal_visualization/builders/imaging_builders.py index 0ce2161..d826048 100644 --- a/src/portal_visualization/builders/imaging_builders.py +++ b/src/portal_visualization/builders/imaging_builders.py @@ -22,6 +22,7 @@ class AbstractImagingViewConfBuilder(ViewConfBuilder): def __init__(self, entity, groups_token, assets_endpoint, **kwargs): self.image_pyramid_regex = None + self.seg_image_pyramid_regex = None self.use_full_resolution = [] self.use_physical_size_scaling = False super().__init__(entity, groups_token, assets_endpoint, **kwargs) @@ -78,9 +79,24 @@ def _get_img_and_offset_url_seg(self, img_path, img_dir): ) ), ) + + def _add_segmentation_image(self, dataset): + file_paths_found = self._get_file_paths() + found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) + filtered_images = [img for img in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img] + + if not filtered_images: + raise FileNotFoundError(f"Segmentation assay with uuid {self._uuid} has no matching files") + + img_url, offsets_url = self._get_img_and_offset_url(filtered_images[0], self.seg_image_pyramid_regex) + dataset.add_object( + ObsSegmentationsOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, obs_types_from_channel_names=True, + # coordinate_transformations=[{"type": "scale", "scale": [0.377.,0.377,1,1,1]}] # need to read from a file + ) + ) def _setup_view_config(self, vc, dataset, view_type, disable_3d=[], use_full_resolution=[]): - if view_type == "raster": + if view_type == "image": vc.add_view(cm.SPATIAL, dataset=dataset, x=3, y=0, w=9, h=12).set_props( useFullResolutionImage=use_full_resolution ) @@ -98,7 +114,6 @@ def _setup_view_config(self, vc, dataset, view_type, disable_3d=[], use_full_res # Adding the segmentation mask on top of the image if view_type == 'kaggle_seg': vc.link_views_by_dict([spatial_view, lc_view], { - # Neutralizing the base-image colors 'imageLayer': CL([{'photometricInterpretation': 'RGB', }]), }, meta=True, scope_prefix=get_initial_coordination_scope_prefix("A", "image")) @@ -121,39 +136,22 @@ def get_conf_cells_common(self, get_img_and_offset_url_func, **kwargs): ImageOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, name=Path(found_images[0]).name) ) if self.view_type == 'kaggle-seg': - file_paths_found = self._get_file_paths() - found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) - filtered_images = [img_path for img_path in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img_path] - if len(filtered_images) == 0: # pragma: no cover - message = f"Image pyramid assay with uuid {self._uuid} has no matching files" - raise FileNotFoundError(message) - - elif len(filtered_images) >= 1: - img_url, offsets_url = self._get_img_and_offset_url( - filtered_images[0], self.seg_image_pyramid_regex - ) - dataset = dataset.add_object( - ObsSegmentationsOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, obs_types_from_channel_names=True)) + self._add_segmentation_image(dataset) + else: - images = [] - for img_path in found_images: - img_url, offsets_url = get_img_and_offset_url_func( - img_path, self.image_pyramid_regex - ) - images.append( - OmeTiffWrapper( - img_url=img_url, offsets_url=offsets_url, name=Path(img_path).name - ) - ) - dataset = dataset.add_object( - MultiImageWrapper( - images, - use_physical_size_scaling=self.use_physical_size_scaling + images = [ + OmeTiffWrapper( + img_url=img_url, offsets_url=offsets_url, name=Path(img_path).name ) + for img_path in found_images + for img_url, offsets_url in [get_img_and_offset_url_func(img_path, self.image_pyramid_regex)] + ] + dataset.add_object( + MultiImageWrapper(images, use_physical_size_scaling=self.use_physical_size_scaling) ) conf = self._setup_view_config(vc, dataset, self.view_type, use_full_resolution=self.use_full_resolution).to_dict() - if "raster" in self.view_type: + if "image" in self.view_type: del conf["datasets"][0]["files"][0]["options"]["renderLayers"] return get_conf_cells(conf) @@ -166,7 +164,7 @@ class ImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) self.image_pyramid_regex = IMAGE_PYRAMID_DIR - self.view_type = "raster" + self.view_type = "image" def get_conf_cells(self, **kwargs): return self.get_conf_cells_common(self._get_img_and_offset_url, **kwargs) From b637a26facafcbb4fffa1fc2b74b74c1bfa35aa6 Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Thu, 9 Jan 2025 09:17:34 -0500 Subject: [PATCH 03/10] Using globals for view-types --- .../builders/imaging_builders.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/portal_visualization/builders/imaging_builders.py b/src/portal_visualization/builders/imaging_builders.py index d826048..a8af98d 100644 --- a/src/portal_visualization/builders/imaging_builders.py +++ b/src/portal_visualization/builders/imaging_builders.py @@ -18,6 +18,9 @@ SEGMENTATION_SUBDIR) from .base_builders import ViewConfBuilder +BASE_IMAGE_VIEW_TYPE = 'image' +SEG_IMAGE_VIEW_TYPE = 'seg' +KAGGLE_IMAGE_VIEW_TYPE = 'kaggle-seg' class AbstractImagingViewConfBuilder(ViewConfBuilder): def __init__(self, entity, groups_token, assets_endpoint, **kwargs): @@ -96,7 +99,7 @@ def _add_segmentation_image(self, dataset): ) def _setup_view_config(self, vc, dataset, view_type, disable_3d=[], use_full_resolution=[]): - if view_type == "image": + if view_type == BASE_IMAGE_VIEW_TYPE: vc.add_view(cm.SPATIAL, dataset=dataset, x=3, y=0, w=9, h=12).set_props( useFullResolutionImage=use_full_resolution ) @@ -112,10 +115,12 @@ def _setup_view_config(self, vc, dataset, view_type, disable_3d=[], use_full_res disable3d=disable_3d, disableChannelsIfRgbDetected=True ) # Adding the segmentation mask on top of the image - if view_type == 'kaggle_seg': + if view_type == KAGGLE_IMAGE_VIEW_TYPE: + # vc.link_views_by_dict([spatial_view, lc_view]) + # TODO: The image-channel view disappears after the following vc.link_views_by_dict([spatial_view, lc_view], { - 'imageLayer': CL([{'photometricInterpretation': 'RGB', }]), - }, meta=True, scope_prefix=get_initial_coordination_scope_prefix("A", "image")) + 'imageLayer': CL([{'photometricInterpretation': 'RGB', }]), + }, meta=True, scope_prefix=get_initial_coordination_scope_prefix("A", "image")) return vc @@ -135,10 +140,9 @@ def get_conf_cells_common(self, get_img_and_offset_url_func, **kwargs): dataset = dataset.add_object( ImageOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, name=Path(found_images[0]).name) ) - if self.view_type == 'kaggle-seg': + if self.view_type == KAGGLE_IMAGE_VIEW_TYPE: self._add_segmentation_image(dataset) - else: images = [ OmeTiffWrapper( @@ -151,7 +155,7 @@ def get_conf_cells_common(self, get_img_and_offset_url_func, **kwargs): MultiImageWrapper(images, use_physical_size_scaling=self.use_physical_size_scaling) ) conf = self._setup_view_config(vc, dataset, self.view_type, use_full_resolution=self.use_full_resolution).to_dict() - if "image" in self.view_type: + if self.view_type == BASE_IMAGE_VIEW_TYPE: del conf["datasets"][0]["files"][0]["options"]["renderLayers"] return get_conf_cells(conf) @@ -164,7 +168,7 @@ class ImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) self.image_pyramid_regex = IMAGE_PYRAMID_DIR - self.view_type = "image" + self.view_type = BASE_IMAGE_VIEW_TYPE def get_conf_cells(self, **kwargs): return self.get_conf_cells_common(self._get_img_and_offset_url, **kwargs) @@ -178,7 +182,7 @@ class SegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) self.image_pyramid_regex = f"{SEGMENTATION_SUBDIR}/{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}" - self.view_type = "seg" + self.view_type = SEG_IMAGE_VIEW_TYPE def get_conf_cells(self, **kwargs): return self.get_conf_cells_common(self._get_img_and_offset_url_seg, **kwargs) @@ -189,7 +193,7 @@ def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) self.image_pyramid_regex = f"{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}" self.seg_image_pyramid_regex = IMAGE_PYRAMID_DIR - self.view_type = "kaggle-seg" + self.view_type = KAGGLE_IMAGE_VIEW_TYPE def get_conf_cells(self, **kwargs): return self.get_conf_cells_common(self._get_img_and_offset_url_seg, **kwargs) From 2658d4febf99ee8f7efcd6ba2ae0567c2c705f2b Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Thu, 9 Jan 2025 09:47:42 -0500 Subject: [PATCH 04/10] Added tests and linting --- .../builders/imaging_builders.py | 54 +++++++---- .../builders/sprm_builders.py | 2 +- src/portal_visualization/client.py | 2 +- src/vis-preview.py | 26 ++--- .../23a25976beb8c02ab589b13a05b28c55.json | 4 + .../df7cac7cb67a822f7007b57c4d8f5e7d.json | 2 +- .../fake-conf.json | 97 +++++++++++++++++++ .../fake-entity.json | 28 ++++++ .../SegmentationMaskBuilder/fake-entity.json | 2 +- test/test_builders.py | 8 +- 10 files changed, 184 insertions(+), 41 deletions(-) create mode 100644 test/assaytype-fixtures/23a25976beb8c02ab589b13a05b28c55.json create mode 100644 test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-conf.json create mode 100644 test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-entity.json diff --git a/src/portal_visualization/builders/imaging_builders.py b/src/portal_visualization/builders/imaging_builders.py index a8af98d..66ee199 100644 --- a/src/portal_visualization/builders/imaging_builders.py +++ b/src/portal_visualization/builders/imaging_builders.py @@ -22,12 +22,14 @@ SEG_IMAGE_VIEW_TYPE = 'seg' KAGGLE_IMAGE_VIEW_TYPE = 'kaggle-seg' + class AbstractImagingViewConfBuilder(ViewConfBuilder): def __init__(self, entity, groups_token, assets_endpoint, **kwargs): self.image_pyramid_regex = None self.seg_image_pyramid_regex = None self.use_full_resolution = [] self.use_physical_size_scaling = False + self.view_type = BASE_IMAGE_VIEW_TYPE super().__init__(entity, groups_token, assets_endpoint, **kwargs) def _get_img_and_offset_url(self, img_path, img_dir): @@ -71,7 +73,7 @@ def _get_img_and_offset_url_seg(self, img_path, img_dir): """ img_url = self._build_assets_url(img_path) - offsets_path= re.sub(IMAGE_PYRAMID_DIR, OFFSETS_DIR, img_dir) + offsets_path = re.sub(IMAGE_PYRAMID_DIR, OFFSETS_DIR, img_dir) return ( img_url, str( @@ -82,21 +84,22 @@ def _get_img_and_offset_url_seg(self, img_path, img_dir): ) ), ) - - def _add_segmentation_image(self, dataset): - file_paths_found = self._get_file_paths() - found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) - filtered_images = [img for img in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img] - if not filtered_images: - raise FileNotFoundError(f"Segmentation assay with uuid {self._uuid} has no matching files") - - img_url, offsets_url = self._get_img_and_offset_url(filtered_images[0], self.seg_image_pyramid_regex) - dataset.add_object( - ObsSegmentationsOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, obs_types_from_channel_names=True, - # coordinate_transformations=[{"type": "scale", "scale": [0.377.,0.377,1,1,1]}] # need to read from a file - ) - ) + def _add_segmentation_image(self, dataset): + file_paths_found = self._get_file_paths() + found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) + filtered_images = [img for img in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img] + + if not filtered_images: + raise FileNotFoundError(f"Segmentation assay with uuid {self._uuid} has no matching files") + + img_url, offsets_url = self._get_img_and_offset_url(filtered_images[0], self.seg_image_pyramid_regex) + dataset.add_object( + ObsSegmentationsOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, obs_types_from_channel_names=True, + # coordinate_transformations=[{"type": "scale", "scale": + # [0.377.,0.377,1,1,1]}] # need to read from a file + ) + ) def _setup_view_config(self, vc, dataset, view_type, disable_3d=[], use_full_resolution=[]): if view_type == BASE_IMAGE_VIEW_TYPE: @@ -142,7 +145,7 @@ def get_conf_cells_common(self, get_img_and_offset_url_func, **kwargs): ) if self.view_type == KAGGLE_IMAGE_VIEW_TYPE: self._add_segmentation_image(dataset) - + else: images = [ OmeTiffWrapper( @@ -154,7 +157,11 @@ def get_conf_cells_common(self, get_img_and_offset_url_func, **kwargs): dataset.add_object( MultiImageWrapper(images, use_physical_size_scaling=self.use_physical_size_scaling) ) - conf = self._setup_view_config(vc, dataset, self.view_type, use_full_resolution=self.use_full_resolution).to_dict() + conf = self._setup_view_config( + vc, + dataset, + self.view_type, + use_full_resolution=self.use_full_resolution).to_dict() if self.view_type == BASE_IMAGE_VIEW_TYPE: del conf["datasets"][0]["files"][0]["options"]["renderLayers"] return get_conf_cells(conf) @@ -165,6 +172,7 @@ class ImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): i.e for high resolution viz-lifted imaging datasets like https://portal.hubmapconsortium.org/browse/dataset/dc289471333309925e46ceb9bafafaf4 """ + def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) self.image_pyramid_regex = IMAGE_PYRAMID_DIR @@ -179,6 +187,7 @@ class SegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): i.e for high resolution viz-lifted imaging datasets like https://portal.hubmapconsortium.org/browse/dataset/ """ + def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) self.image_pyramid_regex = f"{SEGMENTATION_SUBDIR}/{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}" @@ -186,7 +195,8 @@ def __init__(self, entity, groups_token, assets_endpoint, **kwargs): def get_conf_cells(self, **kwargs): return self.get_conf_cells_common(self._get_img_and_offset_url_seg, **kwargs) - + + class KaggleSegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): # The difference from EPIC segmentation is only the file path and transformations def __init__(self, entity, groups_token, assets_endpoint, **kwargs): @@ -264,9 +274,10 @@ def get_conf_cells(self, **kwargs): ) ) dataset = dataset.add_object(MultiImageWrapper(image_wrappers)) - vc = self._setup_view_config_raster( + vc = self._setup_view_config( vc, dataset, + self.view_type, disable_3d=[self._get_hybcycle(img_path) for img_path in sorted_images] ) conf = vc.to_dict() @@ -282,7 +293,8 @@ def _get_pos_name(self, image_path): return re.search(SEQFISH_FILE_REGEX, image_path)[0].split(".")[ 0 ] - + + def get_found_images(image_pyramid_regex, file_paths_found): found_images = [ path for path in get_matches( @@ -290,4 +302,4 @@ def get_found_images(image_pyramid_regex, file_paths_found): ) if 'separate/' not in path ] - return found_images \ No newline at end of file + return found_images diff --git a/src/portal_visualization/builders/sprm_builders.py b/src/portal_visualization/builders/sprm_builders.py index 06c3d59..f59467a 100644 --- a/src/portal_visualization/builders/sprm_builders.py +++ b/src/portal_visualization/builders/sprm_builders.py @@ -118,7 +118,7 @@ def get_conf_cells(self, **kwargs): if self._files[0]["rel_path"] not in file_paths_found: # This tile has no segmentations, # so only show Spatial component without cells sets, genes etc. - vc = self._setup_view_config_raster(vc, dataset, disable_3d=[self._image_name]) + vc = self._setup_view_config(vc, dataset, self.view_type, disable_3d=[self._image_name]) else: # This tile has segmentations so show the analysis results. for file in self._files: diff --git a/src/portal_visualization/client.py b/src/portal_visualization/client.py index 82ad008..3d20e65 100644 --- a/src/portal_visualization/client.py +++ b/src/portal_visualization/client.py @@ -247,7 +247,7 @@ def get_vitessce_conf_cells_and_lifted_uuid( else: # pragma: no cover # We have separate tests for the builder logic try: def get_entity(entity): - if (type(entity) is str): + if (isinstance(entity, str)): return self.get_entity(uuid=entity) return self.get_entity(uuid=entity.get('uuid')) Builder = get_view_config_builder(entity, get_entity, parent, epic_uuid) diff --git a/src/vis-preview.py b/src/vis-preview.py index b528641..ce266e1 100755 --- a/src/vis-preview.py +++ b/src/vis-preview.py @@ -13,6 +13,7 @@ from portal_visualization.epic_factory import get_epic_builder defaults = json.load((Path(__file__).parent / 'defaults.json').open()) + def main(): # pragma: no cover assets_default_url = defaults['assets_url'] @@ -92,19 +93,20 @@ def get_headers(token): # pragma: no cover headers['Authorization'] = f'Bearer {token}' return headers + def get_entity(uuid): - try: - response = requests.get(f'{defaults["dataset_url"]}{uuid}.json', headers=headers) - if response.status_code != 200: - print(f"Error: Received status code {response.status_code}") - else: - try: - data = response.json() - return data - except Exception as e: - print(f"Error in parsing the response {str(e)}") - except Exception as e: - print(f"Error accessing {defaults['assaytypes_url']}{uuid}: {str(e)}") + try: + response = requests.get(f'{defaults["dataset_url"]}{uuid}.json', headers=headers) + if response.status_code != 200: + print(f"Error: Received status code {response.status_code}") + else: + try: + data = response.json() + return data + except Exception as e: + print(f"Error in parsing the response {str(e)}") + except Exception as e: + print(f"Error accessing {defaults['assaytypes_url']}{uuid}: {str(e)}") def get_entity_from_args(url_arg, json_arg, headers): # pragma: no cover diff --git a/test/assaytype-fixtures/23a25976beb8c02ab589b13a05b28c55.json b/test/assaytype-fixtures/23a25976beb8c02ab589b13a05b28c55.json new file mode 100644 index 0000000..684fceb --- /dev/null +++ b/test/assaytype-fixtures/23a25976beb8c02ab589b13a05b28c55.json @@ -0,0 +1,4 @@ +{ + "soft_assaytype": "h-and-e", + "vitessce-hints": ["segmentation_mask", "pyramid", "is_image"] +} diff --git a/test/assaytype-fixtures/df7cac7cb67a822f7007b57c4d8f5e7d.json b/test/assaytype-fixtures/df7cac7cb67a822f7007b57c4d8f5e7d.json index b54b535..6f1fad4 100644 --- a/test/assaytype-fixtures/df7cac7cb67a822f7007b57c4d8f5e7d.json +++ b/test/assaytype-fixtures/df7cac7cb67a822f7007b57c4d8f5e7d.json @@ -1,3 +1,3 @@ { - "vitessce-hints": ["segmentation_mask", "is_image", "pyramid"] + "vitessce-hints": ["segmentation_mask", "is_image", "pyramid", "epic"] } diff --git a/test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-conf.json b/test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-conf.json new file mode 100644 index 0000000..abe151a --- /dev/null +++ b/test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-conf.json @@ -0,0 +1,97 @@ +{ + "version": "1.0.15", + "name": "HuBMAP Data Portal", + "description": "", + "datasets": [ + { + "uid": "A", + "name": "Visualization Files", + "files": [ + { + "fileType": "image.ome-tiff", + "url": "https://example.com/23a25976beb8c02ab589b13a05b28c55/ometiff-pyramids/lab_processed/images/B001_SB-reg005.ome.tif?token=groups_token", + "options": { + "offsetsUrl": "https://example.com/23a25976beb8c02ab589b13a05b28c55/output_offsets/lab_processed/images/B001_SB-reg005.offsets.json?token=groups_token" + } + }, + { + "fileType": "obsSegmentations.ome-tiff", + "url": "https://example.com/23a25976beb8c02ab589b13a05b28c55/ometiff-pyramids/B001_SB-reg005.segmentations.ome.tif?token=groups_token", + "options": { + "obsTypesFromChannelNames": true, + "offsetsUrl": "https://example.com/23a25976beb8c02ab589b13a05b28c55/output_offsets/B001_SB-reg005.segmentations.offsets.json?token=groups_token" + } + } + ] + } + ], + "coordinationSpace": { + "dataset": { + "A": "A" + }, + "imageLayer": { + "init_A_image_0": "__dummy__" + }, + "photometricInterpretation": { + "init_A_image_0": "RGB" + }, + "metaCoordinationScopes": { + "init_A_image_0": { + "imageLayer": [ + "init_A_image_0" + ] + } + }, + "metaCoordinationScopesBy": { + "init_A_image_0": { + "imageLayer": { + "photometricInterpretation": { + "init_A_image_0": "init_A_image_0" + } + } + } + } + }, + "layout": [ + { + "component": "spatialBeta", + "coordinationScopes": { + "dataset": "A", + "metaCoordinationScopes": [ + "init_A_image_0" + ], + "metaCoordinationScopesBy": [ + "init_A_image_0" + ] + }, + "x": 4, + "y": 0, + "w": 8, + "h": 12, + "props": { + "useFullResolutionImage": [] + } + }, + { + "component": "layerControllerBeta", + "coordinationScopes": { + "dataset": "A", + "metaCoordinationScopes": [ + "init_A_image_0" + ], + "metaCoordinationScopesBy": [ + "init_A_image_0" + ] + }, + "x": 0, + "y": 0, + "w": 4, + "h": 8, + "props": { + "disable3d": [], + "disableChannelsIfRgbDetected": true + } + } + ], + "initStrategy": "auto" +} \ No newline at end of file diff --git a/test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-entity.json b/test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-entity.json new file mode 100644 index 0000000..9d3da74 --- /dev/null +++ b/test/good-fixtures/KaggleSegImagePyramidViewConfBuilder/fake-entity.json @@ -0,0 +1,28 @@ +{ + "data_types": ["Histology"], + "status": "QA", + "immediate_ancestors": [ + { + "data_types": ["Histology"] + } + ], + "files": [ + { + "rel_path": "ometiff-pyramids/lab_processed/images/B001_SB-reg005.ome.tif" + }, + { + "rel_path": "output_offsets/lab_processed/images/B001_SB-reg005.offsets.json" + }, + { + "rel_path": "ometiff-pyramids/B001_SB-reg005.segmentations.ome.tif" + }, + { + "rel_path": "output_offsets/B001_SB-reg005.segmentations.offsets.json" + } + ], + "uuid": "23a25976beb8c02ab589b13a05b28c55", + "metadata": { "dag_provenance_list": [] }, + "parent": { "uuid": "8adc3c31ca84ec4b958ed20a7c4f4919" }, + "soft_assaytype": "h-and-e", + "vitessce-hints": ["segmentation_mask", "pyramid", "is_image"] +} diff --git a/test/good-fixtures/SegmentationMaskBuilder/fake-entity.json b/test/good-fixtures/SegmentationMaskBuilder/fake-entity.json index ec85d0c..6d63c7b 100644 --- a/test/good-fixtures/SegmentationMaskBuilder/fake-entity.json +++ b/test/good-fixtures/SegmentationMaskBuilder/fake-entity.json @@ -39,5 +39,5 @@ "uuid": "df7cac7cb67a822f7007b57c4d8f5e7d", "metadata": {"dag_provenance_list": []}, "parent": { "uuid": "22901da5f080b219a514e38381acbb0e" }, - "vitessce-hints": ["segmentation_mask", "is_image", "pyramid"] + "vitessce-hints": ["segmentation_mask", "is_image", "pyramid", "epic"] } \ No newline at end of file diff --git a/test/test_builders.py b/test/test_builders.py index 7f9d076..4ef8abc 100644 --- a/test/test_builders.py +++ b/test/test_builders.py @@ -90,7 +90,7 @@ def test_has_visualization(has_vis_entity): # TODO: Once other epic hints exist, this may need to be adjusted epic_uuid = ( entity.get("uuid") - if "segmentation_mask" in entity.get("vitessce-hints", {}) + if "epic" in entity.get("vitessce-hints", {}) else None ) assert has_vis == has_visualization(entity, get_entity, parent, epic_uuid) @@ -153,7 +153,7 @@ def test_entity_to_vitessce_conf(entity_path, mocker): entity = json.loads(entity_path.read_text()) parent = entity.get("parent") or None # Only used for image pyramids assay_type = get_entity(entity["uuid"]) - if "segmentation_mask" in assay_type["vitessce-hints"]: + if "epic" in assay_type["vitessce-hints"]: epic_uuid = entity.get("uuid") Builder = get_view_config_builder(entity, get_entity, parent, epic_uuid) # Envvars should not be set during normal test runs, @@ -163,10 +163,10 @@ def test_entity_to_vitessce_conf(entity_path, mocker): # epic_uuid = environ.get("EPIC_UUID", "epic_uuid") builder = Builder(entity, groups_token, assets_url) conf, cells = builder.get_conf_cells(marker=marker) - if "segmentation_mask" not in assay_type["vitessce-hints"]: + if "epic" not in assay_type["vitessce-hints"]: assert Builder.__name__ == entity_path.parent.name compare_confs(entity_path, conf, cells) - if "segmentation_mask" in assay_type["vitessce-hints"]: + if "epic" in assay_type["vitessce-hints"]: epic_builder = get_epic_builder(epic_uuid) assert epic_builder is not None assert epic_builder.__name__ == entity_path.parent.name From 629fa39235c13e21c66e22ef218e4aeb2a9db26c Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Thu, 9 Jan 2025 10:10:48 -0500 Subject: [PATCH 05/10] Refactoring and test coverage fix --- .../builders/imaging_builders.py | 14 ++------------ src/portal_visualization/utils.py | 10 ++++++++++ test/test_builders.py | 12 ++++++++++++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/portal_visualization/builders/imaging_builders.py b/src/portal_visualization/builders/imaging_builders.py index 66ee199..7e97b38 100644 --- a/src/portal_visualization/builders/imaging_builders.py +++ b/src/portal_visualization/builders/imaging_builders.py @@ -12,7 +12,7 @@ Component as cm, ) -from ..utils import get_matches, group_by_file_name, get_conf_cells +from ..utils import get_matches, group_by_file_name, get_conf_cells, get_found_images from ..paths import (IMAGE_PYRAMID_DIR, OFFSETS_DIR, SEQFISH_HYB_CYCLE_REGEX, SEQFISH_FILE_REGEX, SEGMENTATION_SUPPORT_IMAGE_SUBDIR, SEGMENTATION_SUBDIR) @@ -90,7 +90,7 @@ def _add_segmentation_image(self, dataset): found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) filtered_images = [img for img in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img] - if not filtered_images: + if not filtered_images: # pragma: no cover raise FileNotFoundError(f"Segmentation assay with uuid {self._uuid} has no matching files") img_url, offsets_url = self._get_img_and_offset_url(filtered_images[0], self.seg_image_pyramid_regex) @@ -293,13 +293,3 @@ def _get_pos_name(self, image_path): return re.search(SEQFISH_FILE_REGEX, image_path)[0].split(".")[ 0 ] - - -def get_found_images(image_pyramid_regex, file_paths_found): - found_images = [ - path for path in get_matches( - file_paths_found, image_pyramid_regex + r".*\.ome\.tiff?$", - ) - if 'separate/' not in path - ] - return found_images diff --git a/src/portal_visualization/utils.py b/src/portal_visualization/utils.py index 2b62eed..cc08a76 100644 --- a/src/portal_visualization/utils.py +++ b/src/portal_visualization/utils.py @@ -72,6 +72,16 @@ def _get_cells_from_obj(vc_obj): ] +def get_found_images(image_pyramid_regex, file_paths_found): + found_images = [ + path for path in get_matches( + file_paths_found, image_pyramid_regex + r".*\.ome\.tiff?$", + ) + if 'separate/' not in path + ] + return found_images + + def files_from_response(response_json): ''' >>> response_json = {'hits': {'hits': [ diff --git a/test/test_builders.py b/test/test_builders.py index 4ef8abc..f00221c 100644 --- a/test/test_builders.py +++ b/test/test_builders.py @@ -9,6 +9,7 @@ import pytest import zarr +from src.portal_visualization.utils import get_found_images from src.portal_visualization.epic_factory import get_epic_builder from src.portal_visualization.builders.base_builders import ConfCells from src.portal_visualization.builder_factory import ( @@ -96,6 +97,17 @@ def test_has_visualization(has_vis_entity): assert has_vis == has_visualization(entity, get_entity, parent, epic_uuid) +def test_get_found_images(): + file_paths = [ + "image_pyramid/sample.ome.tiff", + "image_pyramid/sample_separate/sample.ome.tiff", + ] + regex = "image_pyramid" + result = get_found_images(regex, file_paths) + assert len(result) == 1 + assert result[0] == "image_pyramid/sample.ome.tiff" + + def mock_zarr_store(entity_path, mocker): # Need to mock zarr.open to yield correct values for different scenarios z = zarr.open() From d62f29bb8d106f67075be6306145c63c388f30a1 Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Thu, 9 Jan 2025 10:21:11 -0500 Subject: [PATCH 06/10] Removed coverage from vis-preview --- src/vis-preview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vis-preview.py b/src/vis-preview.py index ce266e1..00938f1 100755 --- a/src/vis-preview.py +++ b/src/vis-preview.py @@ -94,7 +94,7 @@ def get_headers(token): # pragma: no cover return headers -def get_entity(uuid): +def get_entity(uuid): # pragma: no cover try: response = requests.get(f'{defaults["dataset_url"]}{uuid}.json', headers=headers) if response.status_code != 200: From 53ca4aa18d93b4d08085bc2c470dcafe48177b4f Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Mon, 13 Jan 2025 16:39:36 -0500 Subject: [PATCH 07/10] Added example datasets and renamed EPIC image builder --- src/portal_visualization/builder_factory.py | 4 ++-- .../builders/imaging_builders.py | 16 +++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/portal_visualization/builder_factory.py b/src/portal_visualization/builder_factory.py index 93c43c8..787b055 100644 --- a/src/portal_visualization/builder_factory.py +++ b/src/portal_visualization/builder_factory.py @@ -8,7 +8,7 @@ SeqFISHViewConfBuilder, IMSViewConfBuilder, ImagePyramidViewConfBuilder, - SegImagePyramidViewConfBuilder, + EpicSegImagePyramidViewConfBuilder, KaggleSegImagePyramidViewConfBuilder, NanoDESIViewConfBuilder, ) @@ -77,7 +77,7 @@ def get_view_config_builder(entity, get_entity, parent=None, epic_uuid=None): if parent is not None: # TODO: For now epic (base image's) support datasets doesn't have any hints if is_seg_mask and epic_uuid: - return SegImagePyramidViewConfBuilder + return EpicSegImagePyramidViewConfBuilder elif is_seg_mask: return KaggleSegImagePyramidViewConfBuilder diff --git a/src/portal_visualization/builders/imaging_builders.py b/src/portal_visualization/builders/imaging_builders.py index 7e97b38..2ec800b 100644 --- a/src/portal_visualization/builders/imaging_builders.py +++ b/src/portal_visualization/builders/imaging_builders.py @@ -90,7 +90,7 @@ def _add_segmentation_image(self, dataset): found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) filtered_images = [img for img in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img] - if not filtered_images: # pragma: no cover + if not filtered_images: raise FileNotFoundError(f"Segmentation assay with uuid {self._uuid} has no matching files") img_url, offsets_url = self._get_img_and_offset_url(filtered_images[0], self.seg_image_pyramid_regex) @@ -182,10 +182,10 @@ def get_conf_cells(self, **kwargs): return self.get_conf_cells_common(self._get_img_and_offset_url, **kwargs) -class SegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): - """Wrapper class for creating a standard view configuration for image pyramids for segmenation mask, +class EpicSegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): + """Wrapper class for creating a standard view configuration for image pyramids for EPIC segmentation mask, i.e for high resolution viz-lifted imaging datasets like - https://portal.hubmapconsortium.org/browse/dataset/ + https://portal.dev.hubmapconsortium.org/browse/dataset/df7cac7cb67a822f7007b57c4d8f5e7d """ def __init__(self, entity, groups_token, assets_endpoint, **kwargs): @@ -198,7 +198,13 @@ def get_conf_cells(self, **kwargs): class KaggleSegImagePyramidViewConfBuilder(AbstractImagingViewConfBuilder): - # The difference from EPIC segmentation is only the file path and transformations + """Wrapper class for creating a standard view configuration for image pyramids for kaggle-2 datasets, that show, + segmentation mask layered over a base image-pyramid, however, the file structure is different than + EPIC segmentation masks (EpicSegImagePyramidViewConfBuilder) + i.e for high resolution viz-lifted imaging datasets like + https://portal.dev.hubmapconsortium.org/browse/dataset/534a590d7336aa99c7fc7afd41e995fc + """ + def __init__(self, entity, groups_token, assets_endpoint, **kwargs): super().__init__(entity, groups_token, assets_endpoint, **kwargs) self.image_pyramid_regex = f"{IMAGE_PYRAMID_DIR}/{SEGMENTATION_SUPPORT_IMAGE_SUBDIR}" From 9da837203395a71c69ff1620741d8777ac9e3c8a Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Tue, 14 Jan 2025 08:38:33 -0500 Subject: [PATCH 08/10] Added tests for seg_images functions --- .../builders/imaging_builders.py | 23 +++-- test/test_builders.py | 88 ++++++++++++++++--- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/src/portal_visualization/builders/imaging_builders.py b/src/portal_visualization/builders/imaging_builders.py index 2ec800b..51decf8 100644 --- a/src/portal_visualization/builders/imaging_builders.py +++ b/src/portal_visualization/builders/imaging_builders.py @@ -87,19 +87,28 @@ def _get_img_and_offset_url_seg(self, img_path, img_dir): def _add_segmentation_image(self, dataset): file_paths_found = self._get_file_paths() - found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) + if self.seg_image_pyramid_regex is None: + raise ValueError("seg_image_pyramid_regex is not set. Cannot find segmentation images.") + + try: + found_images = get_found_images(self.seg_image_pyramid_regex, file_paths_found) + except Exception as e: + raise RuntimeError(f"Error while searching for segmentation images: {e}") + filtered_images = [img for img in found_images if SEGMENTATION_SUPPORT_IMAGE_SUBDIR not in img] if not filtered_images: raise FileNotFoundError(f"Segmentation assay with uuid {self._uuid} has no matching files") img_url, offsets_url = self._get_img_and_offset_url(filtered_images[0], self.seg_image_pyramid_regex) - dataset.add_object( - ObsSegmentationsOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, obs_types_from_channel_names=True, - # coordinate_transformations=[{"type": "scale", "scale": - # [0.377.,0.377,1,1,1]}] # need to read from a file - ) - ) + if dataset is not None: + dataset.add_object( + ObsSegmentationsOmeTiffWrapper(img_url=img_url, offsets_url=offsets_url, + obs_types_from_channel_names=True, + # coordinate_transformations=[{"type": "scale", "scale": + # [0.377.,0.377,1,1,1]}] # need to read from a file + ) + ) def _setup_view_config(self, vc, dataset, view_type, disable_3d=[], use_full_resolution=[]): if view_type == BASE_IMAGE_VIEW_TYPE: diff --git a/test/test_builders.py b/test/test_builders.py index f00221c..849cc77 100644 --- a/test/test_builders.py +++ b/test/test_builders.py @@ -5,6 +5,7 @@ from pathlib import Path from os import environ from dataclasses import dataclass +from unittest.mock import patch import pytest import zarr @@ -12,10 +13,15 @@ from src.portal_visualization.utils import get_found_images from src.portal_visualization.epic_factory import get_epic_builder from src.portal_visualization.builders.base_builders import ConfCells +from src.portal_visualization.builders.imaging_builders import KaggleSegImagePyramidViewConfBuilder from src.portal_visualization.builder_factory import ( get_view_config_builder, has_visualization, ) +from src.portal_visualization.paths import IMAGE_PYRAMID_DIR + +groups_token = environ.get("GROUPS_TOKEN", "groups_token") +assets_url = environ.get("ASSETS_URL", "https://example.com") def str_presenter(dumper, data): @@ -97,17 +103,6 @@ def test_has_visualization(has_vis_entity): assert has_vis == has_visualization(entity, get_entity, parent, epic_uuid) -def test_get_found_images(): - file_paths = [ - "image_pyramid/sample.ome.tiff", - "image_pyramid/sample_separate/sample.ome.tiff", - ] - regex = "image_pyramid" - result = get_found_images(regex, file_paths) - assert len(result) == 1 - assert result[0] == "image_pyramid/sample.ome.tiff" - - def mock_zarr_store(entity_path, mocker): # Need to mock zarr.open to yield correct values for different scenarios z = zarr.open() @@ -170,8 +165,6 @@ def test_entity_to_vitessce_conf(entity_path, mocker): Builder = get_view_config_builder(entity, get_entity, parent, epic_uuid) # Envvars should not be set during normal test runs, # but to test the end-to-end integration, they are useful. - groups_token = environ.get("GROUPS_TOKEN", "groups_token") - assets_url = environ.get("ASSETS_URL", "https://example.com") # epic_uuid = environ.get("EPIC_UUID", "epic_uuid") builder = Builder(entity, groups_token, assets_url) conf, cells = builder.get_conf_cells(marker=marker) @@ -249,6 +242,75 @@ def compare_confs(entity_path, conf, cells): assert yaml.dump(clean_cells(cells)) == yaml.dump(expected_cells) +@pytest.fixture +def mockSegImagePyramidBuilder(): + class MockBuilder(KaggleSegImagePyramidViewConfBuilder): + def _get_file_paths(self): + return [] + + entity = json.loads( + next( + (Path(__file__).parent / "good-fixtures") + .glob("KaggleSegImagePyramidViewConfBuilder/*-entity.json") + ).read_text() + ) + return MockBuilder(entity, groups_token, assets_url) + + +def test_filtered_images_not_found(mockSegImagePyramidBuilder): + mockSegImagePyramidBuilder.seg_image_pyramid_regex = IMAGE_PYRAMID_DIR + try: + mockSegImagePyramidBuilder._add_segmentation_image(None) + except FileNotFoundError as e: + assert str(e) == f"Segmentation assay with uuid {mockSegImagePyramidBuilder._uuid} has no matching files" + + +def test_filtered_images_no_regex(mockSegImagePyramidBuilder): + mockSegImagePyramidBuilder.seg_image_pyramid_regex = None + try: + mockSegImagePyramidBuilder._add_segmentation_image(None) + except ValueError as e: + assert str(e) == "seg_image_pyramid_regex is not set. Cannot find segmentation images." + + +def mock_get_found_images(regex, file_paths): + raise ValueError("Simulated failure in get_found_images") + + +def test_runtime_error_in_add_segmentation_image(mockSegImagePyramidBuilder): + with patch('src.portal_visualization.builders.imaging_builders.get_found_images', + side_effect=mock_get_found_images): + mockSegImagePyramidBuilder.seg_image_pyramid_regex = "image_pyramid" + + with pytest.raises(RuntimeError) as err: + mockSegImagePyramidBuilder._add_segmentation_image(None) + + assert "Error while searching for segmentation images" in str(err.value) + assert "Simulated failure in get_found_images" in str(err.value) + + +def test_find_segmentation_images_runtime_error(): + with pytest.raises(RuntimeError) as e: + try: + raise FileNotFoundError("No files found in the directory") + except Exception as err: + raise RuntimeError(f"Error while searching for segmentation images: {err}") + + assert "Error while searching for segmentation images:" in str(e.value) + assert "No files found in the directory" in str(e.value) + + +def test_get_found_images(): + file_paths = [ + "image_pyramid/sample.ome.tiff", + "image_pyramid/sample_separate/sample.ome.tiff", + ] + regex = "image_pyramid" + result = get_found_images(regex, file_paths) + assert len(result) == 1 + assert result[0] == "image_pyramid/sample.ome.tiff" + + if __name__ == "__main__": # pragma: no cover parser = argparse.ArgumentParser(description="Generate fixtures") parser.add_argument("--input", required=True, type=Path, help="Input JSON path") From fca12e3ff952cc59af1319c1e37bd743cecda693 Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Wed, 15 Jan 2025 16:22:30 -0500 Subject: [PATCH 09/10] Renamed function for consistency --- test/test_builders.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/test_builders.py b/test/test_builders.py index 849cc77..ba177db 100644 --- a/test/test_builders.py +++ b/test/test_builders.py @@ -243,7 +243,7 @@ def compare_confs(entity_path, conf, cells): @pytest.fixture -def mockSegImagePyramidBuilder(): +def mock_seg_image_pyramid_builder(): class MockBuilder(KaggleSegImagePyramidViewConfBuilder): def _get_file_paths(self): return [] @@ -257,18 +257,18 @@ def _get_file_paths(self): return MockBuilder(entity, groups_token, assets_url) -def test_filtered_images_not_found(mockSegImagePyramidBuilder): - mockSegImagePyramidBuilder.seg_image_pyramid_regex = IMAGE_PYRAMID_DIR +def test_filtered_images_not_found(mock_seg_image_pyramid_builder): + mock_seg_image_pyramid_builder.seg_image_pyramid_regex = IMAGE_PYRAMID_DIR try: - mockSegImagePyramidBuilder._add_segmentation_image(None) + mock_seg_image_pyramid_builder._add_segmentation_image(None) except FileNotFoundError as e: - assert str(e) == f"Segmentation assay with uuid {mockSegImagePyramidBuilder._uuid} has no matching files" + assert str(e) == f"Segmentation assay with uuid {mock_seg_image_pyramid_builder._uuid} has no matching files" -def test_filtered_images_no_regex(mockSegImagePyramidBuilder): - mockSegImagePyramidBuilder.seg_image_pyramid_regex = None +def test_filtered_images_no_regex(mock_seg_image_pyramid_builder): + mock_seg_image_pyramid_builder.seg_image_pyramid_regex = None try: - mockSegImagePyramidBuilder._add_segmentation_image(None) + mock_seg_image_pyramid_builder._add_segmentation_image(None) except ValueError as e: assert str(e) == "seg_image_pyramid_regex is not set. Cannot find segmentation images." @@ -277,13 +277,13 @@ def mock_get_found_images(regex, file_paths): raise ValueError("Simulated failure in get_found_images") -def test_runtime_error_in_add_segmentation_image(mockSegImagePyramidBuilder): +def test_runtime_error_in_add_segmentation_image(mock_seg_image_pyramid_builder): with patch('src.portal_visualization.builders.imaging_builders.get_found_images', side_effect=mock_get_found_images): - mockSegImagePyramidBuilder.seg_image_pyramid_regex = "image_pyramid" + mock_seg_image_pyramid_builder.seg_image_pyramid_regex = "image_pyramid" with pytest.raises(RuntimeError) as err: - mockSegImagePyramidBuilder._add_segmentation_image(None) + mock_seg_image_pyramid_builder._add_segmentation_image(None) assert "Error while searching for segmentation images" in str(err.value) assert "Simulated failure in get_found_images" in str(err.value) From 361b948e26058907eac7d9d0508b3ceaa27b5dfd Mon Sep 17 00:00:00 2001 From: Tabassum Kakar Date: Wed, 15 Jan 2025 16:26:21 -0500 Subject: [PATCH 10/10] Version bumped --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 0f82685..6678432 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -0.3.7 +0.3.8