feat: Support frame.depth

mrousavy · Dec 13, 2024 · 230b49d · 230b49d
1 parent 66d0a62
commit 230b49d
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 5 deletions.
diff --git a/package/ios/Core/CameraSessionDelegate.swift b/package/ios/Core/CameraSessionDelegate.swift
@@ -44,7 +44,7 @@ protocol CameraSessionDelegate: AnyObject {
   /**
    Called for every frame (if video or frameProcessor is enabled)
    */
-  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool)
+  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthBuffer: CMSampleBuffer?)
   /**
    Called whenever a QR/Barcode has been scanned. Only if the CodeScanner Output is enabled
    */

diff --git a/package/ios/FrameProcessors/Frame.h b/package/ios/FrameProcessors/Frame.h
@@ -16,13 +16,14 @@ NS_ASSUME_NONNULL_BEGIN
 
 @interface Frame : NSObject
 
-- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored;
+- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored depthData:(nullable CMSampleBufferRef)depth;
 - (instancetype)init NS_UNAVAILABLE;
 
 - (void)incrementRefCount;
 - (void)decrementRefCount;
 
 @property(nonatomic, readonly) CMSampleBufferRef buffer;
+@property(nonatomic, readonly, nullable) CMSampleBufferRef depth;
 @property(nonatomic, readonly) UIImageOrientation orientation;
 
 @property(nonatomic, readonly) NSString* pixelFormat;

diff --git a/package/ios/FrameProcessors/Frame.m b/package/ios/FrameProcessors/Frame.m
@@ -14,14 +14,16 @@ @implementation Frame {
   CMSampleBufferRef _Nonnull _buffer;
   UIImageOrientation _orientation;
   BOOL _isMirrored;
+  CMSampleBufferRef _Nullable _depth;
 }
 
-- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored {
+- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored depthData:(nullable CMSampleBufferRef)depth {
   self = [super init];
   if (self) {
     _buffer = buffer;
     _orientation = orientation;
     _isMirrored = isMirrored;
+    _depth = depth;
   }
   return self;
 }
@@ -47,6 +49,10 @@ - (CMSampleBufferRef)buffer {
   return _buffer;
 }
 
+- (nullable CMSampleBufferRef)depth {
+  return _depth;
+}
+
 - (BOOL)isValid {
   return _buffer != nil && CFGetRetainCount(_buffer) > 0 && CMSampleBufferIsValid(_buffer);
 }

diff --git a/package/ios/FrameProcessors/FrameHostObject.mm b/package/ios/FrameProcessors/FrameHostObject.mm
@@ -74,6 +74,16 @@
   if (name == "planesCount") {
     return jsi::Value((double)_frame.planesCount);
   }
+  if (name == "depth") {
+    if (_frame.depth == nil) {
+      return jsi::Value::undefined();
+    }
+    jsi::Object object(runtime);
+    CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_frame.depth);
+    object.setProperty(runtime, "width", jsi::Value(static_cast<double>(CVPixelBufferGetWidth(imageBuffer))));
+    object.setProperty(runtime, "height", jsi::Value(static_cast<double>(CVPixelBufferGetHeight(imageBuffer))));
+    return object;
+  }
 
   // Internal methods
   if (name == "incrementRefCount") {

diff --git a/package/ios/React/CameraView.swift b/package/ios/React/CameraView.swift
@@ -363,7 +363,7 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat
     ])
   }
 
-  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool) {
+  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthBuffer: CMSampleBuffer?) {
     // Update latest frame that can be used for snapshot capture
     latestVideoFrame = Snapshot(imageBuffer: sampleBuffer, orientation: orientation)
 

diff --git a/package/src/types/Frame.ts b/package/src/types/Frame.ts
@@ -70,7 +70,6 @@ export interface Frame {
   readonly depth?: {
     readonly width: number
     readonly height: number
-    toArrayBuffer(): ArrayBuffer
   }
 
   /**