From 32db67ca0eb8a19ce8a011f1e528acb92ccefec4 Mon Sep 17 00:00:00 2001 From: Laxma Reddy Patlolla Date: Mon, 31 Mar 2025 14:40:17 -0700 Subject: [PATCH 1/3] Made label data optional for inference and adopted other required changes --- .../segformer_image_segmenter_preprocessor.py | 18 ----------- .../segformer_image_segmenter_tests.py | 32 +++++++++++-------- .../convert_segformer_checkpoints.py | 9 +++++- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/keras_hub/src/models/segformer/segformer_image_segmenter_preprocessor.py b/keras_hub/src/models/segformer/segformer_image_segmenter_preprocessor.py index 20d42c58fd..137f3a7811 100644 --- a/keras_hub/src/models/segformer/segformer_image_segmenter_preprocessor.py +++ b/keras_hub/src/models/segformer/segformer_image_segmenter_preprocessor.py @@ -1,5 +1,3 @@ -import keras - from keras_hub.src.api_export import keras_hub_export from keras_hub.src.models.image_segmenter_preprocessor import ( ImageSegmenterPreprocessor, @@ -8,25 +6,9 @@ from keras_hub.src.models.segformer.segformer_image_converter import ( SegFormerImageConverter, ) -from keras_hub.src.utils.tensor_utils import preprocessing_function - -IMAGENET_DEFAULT_MEAN = [0.485, 0.456, 0.406] -IMAGENET_DEFAULT_STD = [0.229, 0.224, 0.225] @keras_hub_export("keras_hub.models.SegFormerImageSegmenterPreprocessor") class SegFormerImageSegmenterPreprocessor(ImageSegmenterPreprocessor): backbone_cls = SegFormerBackbone image_converter_cls = SegFormerImageConverter - - @preprocessing_function - def call(self, x, y=None, sample_weight=None): - if self.image_converter: - x = self.image_converter(x) - if y is not None: - y = self.image_converter(y) - - x = x / 255 - x = (x - IMAGENET_DEFAULT_MEAN) / IMAGENET_DEFAULT_STD - - return keras.utils.pack_x_y_sample_weight(x, y, sample_weight) diff --git a/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py b/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py index b539b27025..d4ac6de817 100644 --- a/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py +++ b/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py @@ -1,35 +1,41 @@ import numpy as np import pytest -from keras import ops from keras_hub.api.models import MiTBackbone from keras_hub.api.models import SegFormerBackbone from keras_hub.api.models import SegFormerImageSegmenter +from keras_hub.api.models import SegFormerImageSegmenterPreprocessor from keras_hub.src.tests.test_case import TestCase class SegFormerTest(TestCase): def setUp(self): image_encoder = MiTBackbone( - depths=[2, 2], + layerwise_depths=[2, 2], image_shape=(224, 224, 3), hidden_dims=[32, 64], num_layers=2, - blockwise_num_heads=[1, 2], - blockwise_sr_ratios=[8, 4], + layerwise_num_heads=[1, 2], + layerwise_sr_ratios=[8, 4], max_drop_path_rate=0.1, - patch_sizes=[7, 3], - strides=[4, 2], + layerwise_patch_sizes=[7, 3], + layerwise_strides=[4, 2], ) projection_filters = 256 + self.preprocessor = SegFormerImageSegmenterPreprocessor() self.backbone = SegFormerBackbone( image_encoder=image_encoder, projection_filters=projection_filters ) self.input_size = 224 - self.input_data = ops.ones((2, self.input_size, self.input_size, 3)) + self.input_data = np.ones((2, self.input_size, self.input_size, 3)) + self.label_data = np.ones((2, self.input_size, self.input_size, 4)) - self.init_kwargs = {"backbone": self.backbone, "num_classes": 4} + self.init_kwargs = { + "backbone": self.backbone, + "num_classes": 4, + "preprocessor": self.preprocessor, + } def test_segformer_segmenter_construction(self): SegFormerImageSegmenter(backbone=self.backbone, num_classes=4) @@ -40,19 +46,19 @@ def test_segformer_call(self): backbone=self.backbone, num_classes=4 ) - images = np.random.uniform(size=(2, 224, 224, 4)) + images = np.random.uniform(size=(2, 224, 224, 3)) segformer_output = segformer(images) segformer_predict = segformer.predict(images) - assert segformer_output.shape == images.shape - assert segformer_predict.shape == images.shape + self.assertAllEqual(segformer_output.shape, (2, 224, 224, 4)) + self.assertAllEqual(segformer_predict.shape, (2, 224, 224, 4)) def test_task(self): self.run_task_test( cls=SegFormerImageSegmenter, init_kwargs={**self.init_kwargs}, - train_data=self.input_data, - expected_output_shape=(2, 224, 224), + train_data=(self.input_data, self.label_data), + expected_output_shape=(2, 224, 224, 4), ) @pytest.mark.large diff --git a/tools/checkpoint_conversion/convert_segformer_checkpoints.py b/tools/checkpoint_conversion/convert_segformer_checkpoints.py index d87d590e71..73b53d5ada 100644 --- a/tools/checkpoint_conversion/convert_segformer_checkpoints.py +++ b/tools/checkpoint_conversion/convert_segformer_checkpoints.py @@ -94,7 +94,14 @@ def main(_): ) num_classes = 150 if "ade20k" in FLAGS.preset else 19 - preprocessor = SegFormerImageSegmenterPreprocessor() + image_converter = keras_hub.layers.ResNetImageConverter( + height=224, + width=224, + mean=[0.485, 0.456, 0.406], + variance=[0.229**2, 0.224**2, 0.225**2], + scale=1 / 255.0, + ) + preprocessor = SegFormerImageSegmenterPreprocessor(image_converter) segformer_segmenter = keras_hub.models.SegFormerImageSegmenter( backbone=segformer_backbone, num_classes=num_classes, From 7cb56b108fed2025b8a95bfb74ef44023bd0a6b0 Mon Sep 17 00:00:00 2001 From: Laxma Reddy Patlolla Date: Thu, 24 Apr 2025 11:26:53 -0700 Subject: [PATCH 2/3] address comments --- .../segformer_image_segmenter_tests.py | 13 ++++++------- .../convert_segformer_checkpoints.py | 19 +++++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py b/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py index 36dc125917..c73159d03d 100644 --- a/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py +++ b/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py @@ -1,7 +1,6 @@ import numpy as np import pytest - from keras_hub.api.models import MiTBackbone from keras_hub.api.models import SegFormerBackbone from keras_hub.api.models import SegFormerImageSegmenter @@ -13,7 +12,7 @@ class SegFormerTest(TestCase): def setUp(self): image_encoder = MiTBackbone( layerwise_depths=[2, 2], - image_shape=(224, 224, 3), + image_shape=(32, 32, 3), hidden_dims=[32, 64], num_layers=2, layerwise_num_heads=[1, 2], @@ -28,7 +27,7 @@ def setUp(self): image_encoder=image_encoder, projection_filters=projection_filters ) - self.input_size = 224 + self.input_size = 32 self.input_data = np.ones((2, self.input_size, self.input_size, 3)) self.label_data = np.ones((2, self.input_size, self.input_size, 4)) @@ -47,19 +46,19 @@ def test_segformer_call(self): backbone=self.backbone, num_classes=4 ) - images = np.random.uniform(size=(2, 224, 224, 3)) + images = np.random.uniform(size=(2, 32, 32, 3)) segformer_output = segformer(images) segformer_predict = segformer.predict(images) - self.assertAllEqual(segformer_output.shape, (2, 224, 224, 4)) - self.assertAllEqual(segformer_predict.shape, (2, 224, 224, 4)) + self.assertAllEqual(segformer_output.shape, (2, 32, 32, 4)) + self.assertAllEqual(segformer_predict.shape, (2, 32, 32, 4)) def test_task(self): self.run_task_test( cls=SegFormerImageSegmenter, init_kwargs={**self.init_kwargs}, train_data=(self.input_data, self.label_data), - expected_output_shape=(2, 224, 224, 4), + expected_output_shape=(2, 32, 32, 4), ) @pytest.mark.large diff --git a/tools/checkpoint_conversion/convert_segformer_checkpoints.py b/tools/checkpoint_conversion/convert_segformer_checkpoints.py index 73b53d5ada..c6bf852037 100644 --- a/tools/checkpoint_conversion/convert_segformer_checkpoints.py +++ b/tools/checkpoint_conversion/convert_segformer_checkpoints.py @@ -1,6 +1,6 @@ # Usage example -# python tools/checkpoint_conversion/convert_mix_transformer.py \ -# --preset "B0_ade_512" +# python tools/checkpoint_conversion/convert_segformer_checkpoints.py \ +# --preset "b0_ade20k_512" import numpy as np from absl import app @@ -94,12 +94,15 @@ def main(_): ) num_classes = 150 if "ade20k" in FLAGS.preset else 19 - image_converter = keras_hub.layers.ResNetImageConverter( - height=224, - width=224, - mean=[0.485, 0.456, 0.406], - variance=[0.229**2, 0.224**2, 0.225**2], - scale=1 / 255.0, + image_converter = keras_hub.layers.SegFormerImageConverter( + height=resolution, + width=resolution, + scale=[ + 1.0 / (0.229 * 255.0), + 1.0 / (0.224 * 255.0), + 1.0 / (0.225 * 255.0), + ], + offset=[-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.225], ) preprocessor = SegFormerImageSegmenterPreprocessor(image_converter) segformer_segmenter = keras_hub.models.SegFormerImageSegmenter( From 37fdf24c622000a1d54c712bc3a3b75894fd2d2e Mon Sep 17 00:00:00 2001 From: Laxma Reddy Patlolla Date: Thu, 24 Apr 2025 13:56:01 -0700 Subject: [PATCH 3/3] Fix format issues --- .../segformer/segformer_image_segmenter_tests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py b/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py index c73159d03d..136351e386 100644 --- a/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py +++ b/keras_hub/src/models/segformer/segformer_image_segmenter_tests.py @@ -1,10 +1,14 @@ import numpy as np import pytest -from keras_hub.api.models import MiTBackbone -from keras_hub.api.models import SegFormerBackbone -from keras_hub.api.models import SegFormerImageSegmenter -from keras_hub.api.models import SegFormerImageSegmenterPreprocessor +from keras_hub.src.models.mit.mit_backbone import MiTBackbone +from keras_hub.src.models.segformer.segformer_backbone import SegFormerBackbone +from keras_hub.src.models.segformer.segformer_image_segmenter import ( + SegFormerImageSegmenter, +) +from keras_hub.src.models.segformer.segformer_image_segmenter_preprocessor import ( # noqa: E501 + SegFormerImageSegmenterPreprocessor, +) from keras_hub.src.tests.test_case import TestCase