[DO NOT MERGE - PROTOTYPE] DATA-4768 - Pass through annotations from Images

components/camera/camera.go

@@ -87,21 +87,22 @@ type Properties struct {
 
 // NamedImage is a struct that associates the source from where the image came from to the Image.
 type NamedImage struct {
-	data       []byte
-	img        image.Image
-	SourceName string
-	mimeType   string
+	data        []byte
+	img         image.Image
+	SourceName  string
+	mimeType    string
+	Annotations data.Annotations
 }
 
 // NamedImageFromBytes constructs a NamedImage from a byte slice, source name, and mime type.
-func NamedImageFromBytes(data []byte, sourceName, mimeType string) (NamedImage, error) {
+func NamedImageFromBytes(data []byte, sourceName, mimeType string, annotations data.Annotations) (NamedImage, error) {
 	if data == nil {
 		return NamedImage{}, fmt.Errorf("must provide image bytes to construct a named image from bytes")
 	}
 	if mimeType == "" {
 		return NamedImage{}, fmt.Errorf("must provide a mime type to construct a named image")
 	}
-	return NamedImage{data: data, SourceName: sourceName, mimeType: mimeType}, nil
+	return NamedImage{data: data, SourceName: sourceName, mimeType: mimeType, Annotations: annotations}, nil
 }
 
 // NamedImageFromImage constructs a NamedImage from an image.Image, source name, and mime type.
@@ -166,7 +167,8 @@ func (ni *NamedImage) MimeType() string {
 
 // ImageMetadata contains useful information about returned image bytes such as its mimetype.
 type ImageMetadata struct {
-	MimeType string
+	MimeType    string
+	Annotations data.Annotations
 }
 
 // A Camera is a resource that can capture frames.
@@ -284,12 +286,14 @@ func GetImageFromGetImages(
 
 	var img image.Image
 	var mimeType string
+	var annotations data.Annotations
 	if sourceName == nil {
 		img, err = namedImages[0].Image(ctx)
 		if err != nil {
 			return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err)
 		}
 		mimeType = namedImages[0].MimeType()
+		annotations = namedImages[0].Annotations
 	} else {
 		for _, i := range namedImages {
 			if i.SourceName == *sourceName {
@@ -298,6 +302,7 @@ func GetImageFromGetImages(
 					return nil, ImageMetadata{}, fmt.Errorf("could not get image from named image: %w", err)
 				}
 				mimeType = i.MimeType()
+				annotations = i.Annotations
 				break
 			}
 		}
@@ -314,7 +319,7 @@ func GetImageFromGetImages(
 	if err != nil {
 		return nil, ImageMetadata{}, fmt.Errorf("could not encode image with encoding %s: %w", mimeType, err)
 	}
-	return imgBytes, ImageMetadata{MimeType: mimeType}, nil
+	return imgBytes, ImageMetadata{MimeType: mimeType, Annotations: annotations}, nil
 }
 
 // GetImagesFromGetImage will be deprecated after RSDK-11726.
@@ -345,7 +350,7 @@ func GetImagesFromGetImage(
 		logger.Warnf("requested mime type %s, but received %s", mimeType, resMimetype)
 	}
 
-	namedImg, err := NamedImageFromBytes(resBytes, "", resMetadata.MimeType)
+	namedImg, err := NamedImageFromBytes(resBytes, "", resMetadata.MimeType, resMetadata.Annotations)
 	if err != nil {
 		return nil, resource.ResponseMetadata{}, fmt.Errorf("could not create named image: %w", err)
 	}

components/camera/camera_test.go

@@ -13,6 +13,7 @@ import (
 	"go.viam.com/utils/artifact"
 
 	"go.viam.com/rdk/components/camera"
+	"go.viam.com/rdk/data"
 	"go.viam.com/rdk/logging"
 	"go.viam.com/rdk/pointcloud"
 	"go.viam.com/rdk/resource"
@@ -635,17 +636,17 @@ func TestNamedImage(t *testing.T) {
 
 	t.Run("NamedImageFromBytes", func(t *testing.T) {
 		t.Run("success", func(t *testing.T) {
-			ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
+			ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
 			test.That(t, err, test.ShouldBeNil)
 			test.That(t, ni.SourceName, test.ShouldEqual, sourceName)
 			test.That(t, ni.MimeType(), test.ShouldEqual, rutils.MimeTypePNG)
 		})
 		t.Run("error on nil data", func(t *testing.T) {
-			_, err := camera.NamedImageFromBytes(nil, sourceName, rutils.MimeTypePNG)
+			_, err := camera.NamedImageFromBytes(nil, sourceName, rutils.MimeTypePNG, data.Annotations{})
 			test.That(t, err, test.ShouldBeError, errors.New("must provide image bytes to construct a named image from bytes"))
 		})
 		t.Run("error on empty mime type", func(t *testing.T) {
-			_, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, "")
+			_, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, "", data.Annotations{})
 			test.That(t, err, test.ShouldBeError, errors.New("must provide a mime type to construct a named image"))
 		})
 	})
@@ -685,7 +686,7 @@ func TestNamedImage(t *testing.T) {
 		})
 
 		t.Run("when only data is populated, it should decode the data and cache it", func(t *testing.T) {
-			ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
+			ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
 			test.That(t, err, test.ShouldBeNil)
 
 			// first call should decode
@@ -707,15 +708,15 @@ func TestNamedImage(t *testing.T) {
 		})
 
 		t.Run("error when data is invalid", func(t *testing.T) {
-			ni, err := camera.NamedImageFromBytes(badBytes, sourceName, rutils.MimeTypePNG)
+			ni, err := camera.NamedImageFromBytes(badBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
 			test.That(t, err, test.ShouldBeNil)
 			_, err = ni.Image(ctx)
 			test.That(t, err, test.ShouldBeError)
 			test.That(t, err.Error(), test.ShouldEqual, "could not decode image config: image: unknown format")
 		})
 
 		t.Run("error when mime type mismatches and decode fails", func(t *testing.T) {
-			ni, err := camera.NamedImageFromBytes(testImgJPEGBytes, sourceName, rutils.MimeTypePNG)
+			ni, err := camera.NamedImageFromBytes(testImgJPEGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
 			test.That(t, err, test.ShouldBeNil)
 			_, err = ni.Image(ctx)
 			test.That(t, err, test.ShouldBeError)
@@ -726,7 +727,7 @@ func TestNamedImage(t *testing.T) {
 			corruptedPNGBytes := append([]byte(nil), testImgPNGBytes...)
 			corruptedPNGBytes[len(corruptedPNGBytes)-5] = 0 // corrupt it
 
-			ni, err := camera.NamedImageFromBytes(corruptedPNGBytes, sourceName, rutils.MimeTypePNG)
+			ni, err := camera.NamedImageFromBytes(corruptedPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
 			test.That(t, err, test.ShouldBeNil)
 			_, err = ni.Image(ctx)
 			test.That(t, err, test.ShouldBeError)
@@ -736,7 +737,7 @@ func TestNamedImage(t *testing.T) {
 
 	t.Run("Bytes method", func(t *testing.T) {
 		t.Run("when data is already populated, it should return the data and cache it", func(t *testing.T) {
-			ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
+			ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
 			test.That(t, err, test.ShouldBeNil)
 			data, err := ni.Bytes(ctx)
 			test.That(t, err, test.ShouldBeNil)
@@ -781,7 +782,7 @@ func TestNamedImage(t *testing.T) {
 	})
 
 	t.Run("MimeType method", func(t *testing.T) {
-		ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG)
+		ni, err := camera.NamedImageFromBytes(testImgPNGBytes, sourceName, rutils.MimeTypePNG, data.Annotations{})
 		test.That(t, err, test.ShouldBeNil)
 		test.That(t, ni.MimeType(), test.ShouldEqual, rutils.MimeTypePNG)
 	})

components/camera/client.go

@@ -237,7 +237,8 @@ func (c *client) Images(
 			// format. We will remove this once we remove the format field from the proto.
 			img.MimeType = utils.FormatToMimeType[img.Format]
 		}
-		namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType)
+
+		namedImg, err := NamedImageFromBytes(img.Image, img.SourceName, img.MimeType, data.AnnotationsFromProto(img.Annotations))
 		if err != nil {
 			return nil, resource.ResponseMetadata{}, err
 		}

components/camera/collectors.go

@@ -130,8 +130,9 @@ func newReadImageCollector(resource interface{}, params data.CollectorParams) (d
 			TimeReceived:  time.Now(),
 		}
 		return data.NewBinaryCaptureResult(ts, []data.Binary{{
-			MimeType: mimeType,
-			Payload:  img,
+			MimeType:    mimeType,
+			Payload:     img,
+			Annotations: metadata.Annotations,
 		}}), nil
 	})
 	return data.NewCollector(cFunc, params)
@@ -162,8 +163,10 @@ func newGetImagesCollector(resource interface{}, params data.CollectorParams) (d
 			if err != nil {
 				return res, data.NewFailedToReadError(params.ComponentName, getImages.String(), err)
 			}
+			annotations := img.Annotations
+			annotations.Classifications = append(annotations.Classifications, data.Classification{Label: img.SourceName})
 			binaries = append(binaries, data.Binary{
-				Annotations: data.Annotations{Classifications: []data.Classification{{Label: img.SourceName}}},
+				Annotations: annotations,
 				Payload:     imgBytes,
 				MimeType:    data.MimeTypeStringToMimeType(img.MimeType()),
 			})

components/camera/server.go

@@ -129,10 +129,11 @@ func (s *serviceServer) GetImages(
 		}
 		format := utils.MimeTypeToFormat[img.MimeType()]
 		imgMes := &pb.Image{
-			SourceName: img.SourceName,
-			Format:     format,
-			MimeType:   img.MimeType(),
-			Image:      imgBytes,
+			SourceName:  img.SourceName,
+			Format:      format,
+			MimeType:    img.MimeType(),
+			Image:       imgBytes,
+			Annotations: img.Annotations.ToProto(),
 		}
 		imagesMessage = append(imagesMessage, imgMes)
 	}

data/collector_types.go

@@ -370,6 +370,37 @@ func (mt Annotations) Empty() bool {
 	return len(mt.BoundingBoxes) == 0 && len(mt.Classifications) == 0
 }
 
+func AnnotationsFromProto(protoAnnotations *dataPB.Annotations) Annotations {
+	if protoAnnotations == nil {
+		return Annotations{}
+	}
+
+	var bboxes []BoundingBox
+	for _, bb := range protoAnnotations.Bboxes {
+		bboxes = append(bboxes, BoundingBox{
+			Label:          bb.Label,
+			Confidence:     bb.Confidence,
+			XMinNormalized: bb.XMinNormalized,
+			XMaxNormalized: bb.XMaxNormalized,
+			YMinNormalized: bb.YMinNormalized,
+			YMaxNormalized: bb.YMaxNormalized,
+		})
+	}
+
+	var classifications []Classification
+	for _, c := range protoAnnotations.Classifications {
+		classifications = append(classifications, Classification{
+			Label:      c.Label,
+			Confidence: c.Confidence,
+		})
+	}
+
+	return Annotations{
+		BoundingBoxes:   bboxes,
+		Classifications: classifications,
+	}
+}
+
 // ToProto converts Annotations to *dataPB.Annotations.
 func (mt Annotations) ToProto() *dataPB.Annotations {
 	if mt.Empty() {

go.mod

@@ -465,3 +465,5 @@ require (
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
 )
+
+replace go.viam.com/api => go.viam.com/api v0.1.482-0.20251015154957-2e1c8df49beb // DATA-4768

go.sum

@@ -1540,8 +1540,8 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-go.viam.com/api v0.1.477 h1:huAOmn3iejrRapzlYSyB3R0S47itXTUkQ3+kt0Yx02I=
-go.viam.com/api v0.1.477/go.mod h1:p/am76zx8SZ74V/F4rEAYQIpHaaLUwJgY2q3Uw3FIWk=
+go.viam.com/api v0.1.482-0.20251015154957-2e1c8df49beb h1:g2gukTeApBlZyNEjUwjKc+uyoGBd5TKOige3S6wH6gI=
+go.viam.com/api v0.1.482-0.20251015154957-2e1c8df49beb/go.mod h1:p/am76zx8SZ74V/F4rEAYQIpHaaLUwJgY2q3Uw3FIWk=
 go.viam.com/test v1.2.4 h1:JYgZhsuGAQ8sL9jWkziAXN9VJJiKbjoi9BsO33TW3ug=
 go.viam.com/test v1.2.4/go.mod h1:zI2xzosHdqXAJ/kFqcN+OIF78kQuTV2nIhGZ8EzvaJI=
 go.viam.com/utils v0.1.173 h1:2decDVTwJ2zhDubWOz1PV4zU8n1cUBXLqWShGdJHGSg=