From 672f6e8fa8dc44fc57490e47194c99e8ce00f707 Mon Sep 17 00:00:00 2001 From: Tahiya Salam Date: Wed, 15 Oct 2025 14:11:08 -0400 Subject: [PATCH 1/5] Update proto --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 98045e31606..cd371cc4820 100644 --- a/go.mod +++ b/go.mod @@ -465,3 +465,5 @@ require ( github.com/ziutek/mymysql v1.5.4 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c ) + +replace go.viam.com/api => go.viam.com/api v0.1.482-0.20251015154957-2e1c8df49beb // DATA-4768 diff --git a/go.sum b/go.sum index 8a8ac9a5ccc..ab52f37c1e0 100644 --- a/go.sum +++ b/go.sum @@ -1540,8 +1540,8 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.viam.com/api v0.1.477 h1:huAOmn3iejrRapzlYSyB3R0S47itXTUkQ3+kt0Yx02I= -go.viam.com/api v0.1.477/go.mod h1:p/am76zx8SZ74V/F4rEAYQIpHaaLUwJgY2q3Uw3FIWk= +go.viam.com/api v0.1.482-0.20251015154957-2e1c8df49beb h1:g2gukTeApBlZyNEjUwjKc+uyoGBd5TKOige3S6wH6gI= +go.viam.com/api v0.1.482-0.20251015154957-2e1c8df49beb/go.mod h1:p/am76zx8SZ74V/F4rEAYQIpHaaLUwJgY2q3Uw3FIWk= go.viam.com/test v1.2.4 h1:JYgZhsuGAQ8sL9jWkziAXN9VJJiKbjoi9BsO33TW3ug= go.viam.com/test v1.2.4/go.mod h1:zI2xzosHdqXAJ/kFqcN+OIF78kQuTV2nIhGZ8EzvaJI= go.viam.com/utils v0.1.173 h1:2decDVTwJ2zhDubWOz1PV4zU8n1cUBXLqWShGdJHGSg= From 26e3cde2fcded3ae6b091128bc7be00ec8d3c103 Mon Sep 17 00:00:00 2001 From: Tahiya Salam Date: Wed, 24 Sep 2025 13:09:16 -0400 Subject: [PATCH 2/5] Change APIs --- components/camera/camera.go | 3 ++- components/camera/collectors.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/camera/camera.go b/components/camera/camera.go index 482fdd578f3..b0d49675705 100644 --- a/components/camera/camera.go +++ b/components/camera/camera.go @@ -166,7 +166,8 @@ func (ni *NamedImage) MimeType() string { // ImageMetadata contains useful information about returned image bytes such as its mimetype. type ImageMetadata struct { - MimeType string + MimeType string + Annotations data.Annotations } // A Camera is a resource that can capture frames. diff --git a/components/camera/collectors.go b/components/camera/collectors.go index f7704be9ee9..c27d620693d 100644 --- a/components/camera/collectors.go +++ b/components/camera/collectors.go @@ -130,8 +130,9 @@ func newReadImageCollector(resource interface{}, params data.CollectorParams) (d TimeReceived: time.Now(), } return data.NewBinaryCaptureResult(ts, []data.Binary{{ - MimeType: mimeType, - Payload: img, + MimeType: mimeType, + Payload: img, + Annotations: metadata.Annotations, }}), nil }) return data.NewCollector(cFunc, params) From 44526be28204d0958f4d01ccb08ac1f25bc0d989 Mon Sep 17 00:00:00 2001 From: Tahiya Salam Date: Mon, 20 Oct 2025 14:28:45 -0400 Subject: [PATCH 3/5] Add logic to GetImages collector --- components/camera/camera.go | 20 ++++++++++++-------- components/camera/camera_test.go | 19 ++++++++++--------- components/camera/client.go | 2 +- components/camera/collectors.go | 4 +++- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/components/camera/camera.go b/components/camera/camera.go index b0d49675705..4b81fc1d263 100644 --- a/components/camera/camera.go +++ b/components/camera/camera.go @@ -87,21 +87,22 @@ type Properties struct { // NamedImage is a struct that associates the source from where the image came from to the Image. type NamedImage struct { - data []byte - img image.Image - SourceName string - mimeType string + data []byte + img image.Image + SourceName string + mimeType string + Annotations data.Annotations } // NamedImageFromBytes constructs a NamedImage from a byte slice, source name, and mime type. -func NamedImageFromBytes(data []byte, sourceName, mimeType string) (NamedImage, error) { +func NamedImageFromBytes(data []byte, sourceName, mimeType string, annotations data.Annotations) (NamedImage, error) { if data == nil { return NamedImage{}, fmt.Errorf("must provide image bytes to construct a named image from bytes") } if mimeType == "" { return NamedImage{}, fmt.Errorf("must provide a mime type to construct a named image") } - return NamedImage{data: data, SourceName: sourceName, mimeType: mimeType}, nil + return NamedImage{data: data, SourceName: sourceName, mimeType: mimeType, Annotations: annotations}, nil } // NamedImageFromImage constructs a NamedImage from an image.Image, source name, and mime type. @@ -285,12 +286,14 @@ func GetImageFromGetImages( var img image.Image var mimeType string + var annotations data.Annotations if sourceName == nil { img, err = namedImages[0].Image(ctx) if err != nil { return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err) } mimeType = namedImages[0].MimeType() + annotations = namedImages[0].Annotations } else { for _, i := range namedImages { if i.SourceName == *sourceName { @@ -299,6 +302,7 @@ func GetImageFromGetImages( return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err) } mimeType = i.MimeType() + annotations = i.Annotations break } } @@ -315,7 +319,7 @@ func GetImageFromGetImages( if err != nil { return nil, ImageMetadata{}, fmt.Errorf("could not encode image with encoding %s: %w", mimeType, err) } - return imgBytes, ImageMetadata{MimeType: mimeType}, nil + return imgBytes, ImageMetadata{MimeType: mimeType, Annotations: annotations}, nil } // GetImagesFromGetImage will be deprecated after RSDK-11726. @@ -346,7 +350,7 @@ func GetImagesFromGetImage( logger.Warnf("requested mime type %s, but received %s", mimeType, resMimetype) } - namedImg, err := NamedImageFromBytes(resBytes, "", resMetadata.MimeType) + namedImg, err := NamedImageFromBytes(resBytes, "", resMetadata.MimeType, resMetadata.Annotations) if err != nil { return nil, resource.ResponseMetadata{}, fmt.Errorf("could not create named image: %w", err) } diff --git a/components/camera/camera_test.go b/components/camera/camera_test.go index 5c3d909e9b2..bada7638dbc 100644 --- a/components/camera/camera_test.go +++ b/components/camera/camera_test.go @@ -13,6 +13,7 @@ import ( "go.viam.com/utils/artifact" "go.viam.com/rdk/components/camera" + "go.viam.com/rdk/data" "go.viam.com/rdk/logging" "go.viam.com/rdk/pointcloud" "go.viam.com/rdk/resource" @@ -635,17 +636,17 @@ func TestNamedImage(t *testing.T) { t.Run("NamedImageFromBytes", func(t *testing.T) { t.Run("success", func(t *testing.T) { - ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG) + ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeNil) test.That(t, ni.SourceName, test.ShouldEqual, sourceName) test.That(t, ni.MimeType(), test.ShouldEqual, rutils.MimeTypePNG) }) t.Run("error on nil data", func(t *testing.T) { - _, err := camera.NamedImageFromBytes(nil, sourceName, rutils.MimeTypePNG) + _, err := camera.NamedImageFromBytes(nil, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeError, errors.New("must provide image bytes to construct a named image from bytes")) }) t.Run("error on empty mime type", func(t *testing.T) { - _, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, "") + _, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, "", data.Annotations{}) test.That(t, err, test.ShouldBeError, errors.New("must provide a mime type to construct a named image")) }) }) @@ -685,7 +686,7 @@ func TestNamedImage(t *testing.T) { }) t.Run("when only data is populated, it should decode the data and cache it", func(t *testing.T) { - ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG) + ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeNil) // first call should decode @@ -707,7 +708,7 @@ func TestNamedImage(t *testing.T) { }) t.Run("error when data is invalid", func(t *testing.T) { - ni, err := camera.NamedImageFromBytes(badBytes, sourceName, rutils.MimeTypePNG) + ni, err := camera.NamedImageFromBytes(badBytes, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeNil) _, err = ni.Image(ctx) test.That(t, err, test.ShouldBeError) @@ -715,7 +716,7 @@ func TestNamedImage(t *testing.T) { }) t.Run("error when mime type mismatches and decode fails", func(t *testing.T) { - ni, err := camera.NamedImageFromBytes(testImgJPEGBytes, sourceName, rutils.MimeTypePNG) + ni, err := camera.NamedImageFromBytes(testImgJPEGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeNil) _, err = ni.Image(ctx) test.That(t, err, test.ShouldBeError) @@ -726,7 +727,7 @@ func TestNamedImage(t *testing.T) { corruptedPNGBytes := append([]byte(nil), testImgPNGBytes...) corruptedPNGBytes[len(corruptedPNGBytes)-5] = 0 // corrupt it - ni, err := camera.NamedImageFromBytes(corruptedPNGBytes, sourceName, rutils.MimeTypePNG) + ni, err := camera.NamedImageFromBytes(corruptedPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeNil) _, err = ni.Image(ctx) test.That(t, err, test.ShouldBeError) @@ -736,7 +737,7 @@ func TestNamedImage(t *testing.T) { t.Run("Bytes method", func(t *testing.T) { t.Run("when data is already populated, it should return the data and cache it", func(t *testing.T) { - ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG) + ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeNil) data, err := ni.Bytes(ctx) test.That(t, err, test.ShouldBeNil) @@ -781,7 +782,7 @@ func TestNamedImage(t *testing.T) { }) t.Run("MimeType method", func(t *testing.T) { - ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG) + ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{}) test.That(t, err, test.ShouldBeNil) test.That(t, ni.MimeType(), test.ShouldEqual, rutils.MimeTypePNG) }) diff --git a/components/camera/client.go b/components/camera/client.go index 66e355c0d3a..911da214461 100644 --- a/components/camera/client.go +++ b/components/camera/client.go @@ -237,7 +237,7 @@ func (c *client) Images( // format. We will remove this once we remove the format field from the proto. img.MimeType = utils.FormatToMimeType[img.Format] } - namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType) + namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType, data.Annotations{}) if err != nil { return nil, resource.ResponseMetadata{}, err } diff --git a/components/camera/collectors.go b/components/camera/collectors.go index c27d620693d..ea8f7d8ade8 100644 --- a/components/camera/collectors.go +++ b/components/camera/collectors.go @@ -163,8 +163,10 @@ func newGetImagesCollector(resource interface{}, params data.CollectorParams) (d if err != nil { return res, data.NewFailedToReadError(params.ComponentName, getImages.String(), err) } + annotations := img.Annotations + annotations.Classifications = append(annotations.Classifications, data.Classification{Label: img.SourceName}) binaries = append(binaries, data.Binary{ - Annotations: data.Annotations{Classifications: []data.Classification{{Label: img.SourceName}}}, + Annotations: annotations, Payload: imgBytes, MimeType: data.MimeTypeStringToMimeType(img.MimeType()), }) From c5e89c40fb147fa1bf0938e6a4003ba5419f107a Mon Sep 17 00:00:00 2001 From: Tahiya Salam Date: Tue, 21 Oct 2025 12:04:17 -0400 Subject: [PATCH 4/5] Pass through in named images --- components/camera/client.go | 3 ++- data/collector_types.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/components/camera/client.go b/components/camera/client.go index 911da214461..e5a582f6389 100644 --- a/components/camera/client.go +++ b/components/camera/client.go @@ -237,7 +237,8 @@ func (c *client) Images( // format. We will remove this once we remove the format field from the proto. img.MimeType = utils.FormatToMimeType[img.Format] } - namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType, data.Annotations{}) + + namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType, data.AnnotationsFromProto(img.Annotations)) if err != nil { return nil, resource.ResponseMetadata{}, err } diff --git a/data/collector_types.go b/data/collector_types.go index 832467b7e81..fe07077fe2a 100644 --- a/data/collector_types.go +++ b/data/collector_types.go @@ -370,6 +370,37 @@ func (mt Annotations) Empty() bool { return len(mt.BoundingBoxes) == 0 && len(mt.Classifications) == 0 } +func AnnotationsFromProto(protoAnnotations *dataPB.Annotations) Annotations { + if protoAnnotations == nil { + return Annotations{} + } + + var bboxes []BoundingBox + for _, bb := range protoAnnotations.Bboxes { + bboxes = append(bboxes, BoundingBox{ + Label: bb.Label, + Confidence: bb.Confidence, + XMinNormalized: bb.XMinNormalized, + XMaxNormalized: bb.XMaxNormalized, + YMinNormalized: bb.YMinNormalized, + YMaxNormalized: bb.YMaxNormalized, + }) + } + + var classifications []Classification + for _, c := range protoAnnotations.Classifications { + classifications = append(classifications, Classification{ + Label: c.Label, + Confidence: c.Confidence, + }) + } + + return Annotations{ + BoundingBoxes: bboxes, + Classifications: classifications, + } +} + // ToProto converts Annotations to *dataPB.Annotations. func (mt Annotations) ToProto() *dataPB.Annotations { if mt.Empty() { From 95be61ac7213dfa154b2eafcffaed56428556078 Mon Sep 17 00:00:00 2001 From: Tahiya Salam Date: Tue, 21 Oct 2025 12:35:59 -0400 Subject: [PATCH 5/5] Add change to server --- components/camera/server.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/camera/server.go b/components/camera/server.go index 933a1559536..7ff4e6b4c8f 100644 --- a/components/camera/server.go +++ b/components/camera/server.go @@ -129,10 +129,11 @@ func (s *serviceServer) GetImages( } format := utils.MimeTypeToFormat[img.MimeType()] imgMes := &pb.Image{ - SourceName: img.SourceName, - Format: format, - MimeType: img.MimeType(), - Image: imgBytes, + SourceName: img.SourceName, + Format: format, + MimeType: img.MimeType(), + Image: imgBytes, + Annotations: img.Annotations.ToProto(), } imagesMessage = append(imagesMessage, imgMes) }