From ce4a08fed80c3ca0574f58061acf4ac7cc70b1ed Mon Sep 17 00:00:00 2001 From: Scott Roy Date: Fri, 5 Jun 2026 11:14:42 -0700 Subject: [PATCH] Add ObjC/Swift bindings for the ImageProcessor (#20051) Summary: This diff adds ObjC/Swift bindings for the image processor. Only the pixelbuffer variants get ObjC/Swift bindings. The process/process_yuv variants are only accessible from C++ for now. Example: ``` import ExecuTorch // Configure once let config = ImageProcessorConfig( targetWidth: 224, targetHeight: 224, normalization: .imagenet() ) let processor = ImageProcessor(config: config) // Process a CVPixelBuffer (BGRA/RGBA/NV12/P010) let tensor: Tensor = try processor.process(pixelBuffer) // Output: [1, 3, 224, 224] Float tensor // For video (reuse tensor to avoid allocations) let output = Tensor([1, 3, 224, 224], Array(repeating: 0, count: 3*224*224)) try processor.process(pixelBuffer, into: output) ``` Differential Revision: D106898406 --- extension/apple/BUCK | 3 + .../Exported/ExecuTorch+ImageProcessor.swift | 96 ++++++++ .../apple/ExecuTorch/Exported/ExecuTorch.h | 1 + .../Exported/ExecuTorchImageProcessor.h | 147 ++++++++++++ .../Exported/ExecuTorchImageProcessor.mm | 219 ++++++++++++++++++ .../__tests__/ImageProcessorTest.swift | 219 ++++++++++++++++++ scripts/build_apple_frameworks.sh | 1 + tools/cmake/preset/apple_common.cmake | 1 + 8 files changed, 687 insertions(+) create mode 100644 extension/apple/ExecuTorch/Exported/ExecuTorch+ImageProcessor.swift create mode 100644 extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.h create mode 100644 extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.mm create mode 100644 extension/apple/ExecuTorch/__tests__/ImageProcessorTest.swift diff --git a/extension/apple/BUCK b/extension/apple/BUCK index 521fff5cd8b..0c04eea9ca1 100644 --- a/extension/apple/BUCK +++ b/extension/apple/BUCK @@ -11,6 +11,7 @@ non_fbcode_target(_kind = fb_apple_library, autoglob_mode = "EXPORT_UNLESS_INTERNAL", extension_api_only = True, frameworks = [ + "CoreVideo", "Foundation", ], preprocessor_flags = [ @@ -29,11 +30,13 @@ non_fbcode_target(_kind = fb_apple_library, visibility = EXECUTORCH_CLIENTS, deps = select({ "ovr_config//os:macos": [ + "//xplat/executorch/extension/image:image_processorAppleMac", "//xplat/executorch/extension/module:moduleAppleMac", "//xplat/executorch/extension/tensor:tensorAppleMac", "//xplat/executorch/runtime/platform:platformAppleMac", ], "DEFAULT": [ + "//xplat/executorch/extension/image:image_processorApple", "//xplat/executorch/extension/module:moduleApple", "//xplat/executorch/extension/tensor:tensorApple", "//xplat/executorch/runtime/platform:platformApple", diff --git a/extension/apple/ExecuTorch/Exported/ExecuTorch+ImageProcessor.swift b/extension/apple/ExecuTorch/Exported/ExecuTorch+ImageProcessor.swift new file mode 100644 index 00000000000..20a793aee3c --- /dev/null +++ b/extension/apple/ExecuTorch/Exported/ExecuTorch+ImageProcessor.swift @@ -0,0 +1,96 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +import CoreVideo + +public extension ImageNormalization { + /// Create a normalization with a custom scale factor and per-channel RGB mean + /// and standard deviation. `mean` and `standardDeviation` must each contain + /// exactly 3 elements (R, G, B); every `standardDeviation` entry must be + /// nonzero. Applied per channel as + /// `(pixel * scaleFactor - mean[c]) / standardDeviation[c]`. + convenience init(scaleFactor: Float, mean: [Float], standardDeviation: [Float]) { + precondition(mean.count == 3, "mean must have exactly 3 elements (R, G, B)") + precondition( + standardDeviation.count == 3, + "standardDeviation must have exactly 3 elements (R, G, B)") + self.init( + __scaleFactor: scaleFactor, + mean: mean.map { NSNumber(value: $0) }, + standardDeviation: standardDeviation.map { NSNumber(value: $0) }) + } +} + +public extension ImageProcessorConfig { + /// Source pixel count (width * height) sentinels for `gpuMinInputPixels`. + static let alwaysGPU = 0 + static let alwaysCPU = Int.max + + /// Create an image processor config, specifying only the values that differ + /// from the defaults. + /// + /// `gpuMinInputPixels` is the minimum source pixel count at which the GPU + /// path may be used; smaller inputs run on the CPU. Use `.alwaysGPU` (0) or + /// `.alwaysCPU` to force a path. + convenience init( + targetWidth: Int, + targetHeight: Int, + resizeMode: ImageResizeMode = .stretch, + letterboxAnchor: ImageLetterboxAnchor = .center, + padValue: Float = 0, + normalization: ImageNormalization = .zeroToOne(), + gpuMinInputPixels: Int = ImageProcessorConfig.defaultGpuMinInputPixels + ) { + self.init( + __targetWidth: targetWidth, + targetHeight: targetHeight, + resizeMode: resizeMode, + letterboxAnchor: letterboxAnchor, + padValue: padValue, + normalization: normalization, + gpuMinInputPixels: gpuMinInputPixels) + } +} + +public extension ImageProcessor { + /// Process a CVPixelBuffer into a normalized float tensor. + /// + /// Auto-detects pixel format from the buffer. Supported formats: BGRA, + /// RGBA, 8-bit NV12, and 10-bit P010. Output is a `Tensor` with + /// shape `[1, 3, target_height, target_width]`. + /// + /// The buffer is treated as already upright: orientation correction is not + /// applied and cannot be derived from a CVPixelBuffer, so the caller is + /// responsible for supplying an upright buffer. + func process(_ pixelBuffer: CVPixelBuffer) throws -> Tensor { + let anyTensor = try processPixelBuffer(pixelBuffer) + return Tensor(anyTensor) + } + + /// Process a CVPixelBuffer into a caller-provided tensor, reusing its storage. + /// + /// Avoids the per-call allocation of `process(_:)`, which matters for + /// sustained video. `tensor` must be a `Tensor` with shape + /// `[1, 3, target_height, target_width]`; its storage is overwritten and can + /// be reused across frames. The contents are valid until the next call that + /// writes into the same tensor. + /// + /// The buffer is treated as already upright (see `process(_:)`). + func process(_ pixelBuffer: CVPixelBuffer, into tensor: Tensor) throws { + try processPixelBuffer(pixelBuffer, into: tensor.anyTensor) + } + + /// Letterbox padding (per side, in pixels) applied for a source of the given + /// size: `x` is the left/right pad and `y` the top/bottom pad of the resized + /// content. Returns `(0, 0)` for the stretch resize mode or the top-left + /// anchor. Lets callers map the padded output back to the source region. + func computeLetterboxPadding(inputWidth: Int, inputHeight: Int) -> (x: Int, y: Int) { + let padding = __computeLetterboxPadding(forInputWidth: inputWidth, height: inputHeight) + return (padding.x, padding.y) + } +} diff --git a/extension/apple/ExecuTorch/Exported/ExecuTorch.h b/extension/apple/ExecuTorch/Exported/ExecuTorch.h index d0ad6c2840a..84ad0512ee3 100644 --- a/extension/apple/ExecuTorch/Exported/ExecuTorch.h +++ b/extension/apple/ExecuTorch/Exported/ExecuTorch.h @@ -9,6 +9,7 @@ #import "ExecuTorchBackendOption.h" #import "ExecuTorchBackendOptionsMap.h" #import "ExecuTorchError.h" +#import "ExecuTorchImageProcessor.h" #import "ExecuTorchLog.h" #import "ExecuTorchModule.h" #import "ExecuTorchTensor.h" diff --git a/extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.h b/extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.h new file mode 100644 index 00000000000..3c8f7a40966 --- /dev/null +++ b/extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import +#import + +#import "ExecuTorchTensor.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef NS_ENUM(uint8_t, ExecuTorchImageResizeMode) { + ExecuTorchImageResizeModeStretch, + ExecuTorchImageResizeModeLetterbox, +} NS_SWIFT_NAME(ImageResizeMode); + +typedef NS_ENUM(uint8_t, ExecuTorchImageLetterboxAnchor) { + ExecuTorchImageLetterboxAnchorCenter, + ExecuTorchImageLetterboxAnchorTopLeft, +} NS_SWIFT_NAME(ImageLetterboxAnchor); + +/// Per-side letterbox padding in pixels: `x` is the left/right pad and `y` the +/// top/bottom pad of the resized content. +typedef struct ExecuTorchImageLetterboxPadding { + NSInteger x; + NSInteger y; +} ExecuTorchImageLetterboxPadding NS_SWIFT_NAME(ImageLetterboxPadding); + +NS_SWIFT_NAME(ImageNormalization) +__attribute__((objc_subclassing_restricted)) +@interface ExecuTorchImageNormalization : NSObject + ++ (instancetype)zeroToOne; ++ (instancetype)imagenet; + +/// Create a normalization with a custom scale factor and per-channel RGB mean +/// and standard deviation. `mean` and `standardDeviation` must each contain +/// exactly 3 elements (R, G, B). Normalization is applied per channel as +/// `(pixel * scaleFactor - mean[c]) / standardDeviation[c]`, so every +/// `standardDeviation` entry must be nonzero. +- (instancetype)initWithScaleFactor:(float)scaleFactor + mean:(NSArray *)mean + standardDeviation:(NSArray *)standardDeviation + NS_REFINED_FOR_SWIFT; + ++ (instancetype)new NS_UNAVAILABLE; +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_SWIFT_NAME(ImageProcessorConfig) +__attribute__((objc_subclassing_restricted)) +@interface ExecuTorchImageProcessorConfig : NSObject + +@property(nonatomic, readonly) NSInteger targetWidth; +@property(nonatomic, readonly) NSInteger targetHeight; +@property(nonatomic, readonly) ExecuTorchImageResizeMode resizeMode; +@property(nonatomic, readonly) ExecuTorchImageLetterboxAnchor letterboxAnchor; +@property(nonatomic, readonly) float padValue; +@property(nonatomic, readonly) ExecuTorchImageNormalization *normalization; +// Minimum source pixel count (width * height) at which the GPU path may be +// used; smaller inputs run on the CPU. 0 forces GPU, NSIntegerMax forces CPU. +@property(nonatomic, readonly) NSInteger gpuMinInputPixels; + +// Default value for gpuMinInputPixels (mirrors the C++ config default). +@property(class, nonatomic, readonly) NSInteger defaultGpuMinInputPixels; + +- (instancetype)initWithTargetWidth:(NSInteger)targetWidth + targetHeight:(NSInteger)targetHeight + resizeMode:(ExecuTorchImageResizeMode)resizeMode + letterboxAnchor:(ExecuTorchImageLetterboxAnchor)letterboxAnchor + padValue:(float)padValue + normalization:(ExecuTorchImageNormalization *)normalization + gpuMinInputPixels:(NSInteger)gpuMinInputPixels NS_REFINED_FOR_SWIFT; + ++ (instancetype)new NS_UNAVAILABLE; +- (instancetype)init NS_UNAVAILABLE; + +@end + +/// Thread-safety: ExecuTorchImageProcessor is NOT thread-safe per instance. +/// Internal scratch buffers are mutated during processing. Use one instance +/// per concurrent caller. Different instances are safe to use concurrently. +NS_SWIFT_NAME(ImageProcessor) +__attribute__((objc_subclassing_restricted)) +@interface ExecuTorchImageProcessor : NSObject + +@property(nonatomic, readonly) ExecuTorchImageProcessorConfig *config; + +- (instancetype)initWithConfig:(ExecuTorchImageProcessorConfig *)config; + +/// Process a CVPixelBuffer into a normalized float tensor. +/// +/// Auto-detects pixel format from the buffer's metadata. Supported +/// formats: BGRA, RGBA, 8-bit NV12, and 10-bit P010 (P010 is narrowed to NV12 +/// internally). Other formats return an error. +/// +/// The buffer is treated as already upright. Orientation correction is not +/// applied and cannot be derived from a CVPixelBuffer, so the caller is +/// responsible for supplying an upright buffer (e.g. by configuring the +/// capture connection's orientation). +/// +/// @param pixelBuffer The input pixel buffer. +/// @param error On failure, set to an NSError describing what went wrong. +/// @return An ExecuTorchTensor with shape [1, 3, H, W] (CHW), or nil on failure. +- (nullable ExecuTorchTensor *)processPixelBuffer:(_Nullable CVPixelBufferRef)pixelBuffer + error:(NSError **)error; + +/// Process a CVPixelBuffer into a caller-provided tensor, reusing its storage. +/// +/// Avoids the per-call output allocation of processPixelBuffer:error:, which +/// matters for sustained video. `tensor` must be a Float tensor shaped +/// [1, 3, targetHeight, targetWidth]; its storage is overwritten and can be +/// reused across frames. The result aliases `tensor`, so the caller must +/// finish using the previous result before the next call. +/// +/// @param pixelBuffer The input pixel buffer. +/// @param tensor The output tensor to fill. +/// @param error On failure, set to an NSError describing what went wrong. +/// @return YES on success, NO on failure. +- (BOOL)processPixelBuffer:(_Nullable CVPixelBufferRef)pixelBuffer + intoTensor:(ExecuTorchTensor *)tensor + error:(NSError **)error; + +/// Letterbox padding (per side, in pixels) the processor applies for a source +/// of the given size: `x` is the left/right pad and `y` the top/bottom pad of +/// the resized content. Returns {0, 0} for the stretch resize mode or the +/// top-left anchor. Lets callers map the padded output back to the source +/// region without replicating the resize geometry. +/// +/// @param inputWidth The source pixel width. +/// @param inputHeight The source pixel height. +/// @return The {x, y} padding in pixels. +- (ExecuTorchImageLetterboxPadding)computeLetterboxPaddingForInputWidth:(NSInteger)inputWidth + height:(NSInteger)inputHeight + NS_REFINED_FOR_SWIFT; + ++ (instancetype)new NS_UNAVAILABLE; +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.mm b/extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.mm new file mode 100644 index 00000000000..c62b3312641 --- /dev/null +++ b/extension/apple/ExecuTorch/Exported/ExecuTorchImageProcessor.mm @@ -0,0 +1,219 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "ExecuTorchImageProcessor.h" + +#import "ExecuTorchError.h" + +#import +#import +#import + +#include + +using executorch::extension::TensorPtr; +using executorch::extension::image::ImageProcessor; +using executorch::extension::image::ImageProcessorConfig; +using executorch::extension::image::LetterboxAnchor; +using executorch::extension::image::Normalization; +using executorch::extension::image::Orientation; +using executorch::extension::image::process_pixelbuffer; +using executorch::extension::image::process_pixelbuffer_into; +using executorch::extension::image::ResizeMode; + +// Verify enum value parity between ObjC and C++ at compile time +static_assert((int)ExecuTorchImageResizeModeStretch == (int)ResizeMode::STRETCH, "ExecuTorchImageResizeModeStretch must match ResizeMode::STRETCH"); +static_assert((int)ExecuTorchImageResizeModeLetterbox == (int)ResizeMode::LETTERBOX, "ExecuTorchImageResizeModeLetterbox must match ResizeMode::LETTERBOX"); +static_assert((int)ExecuTorchImageLetterboxAnchorCenter == (int)LetterboxAnchor::CENTER, "ExecuTorchImageLetterboxAnchorCenter must match LetterboxAnchor::CENTER"); +static_assert((int)ExecuTorchImageLetterboxAnchorTopLeft == (int)LetterboxAnchor::TOP_LEFT, "ExecuTorchImageLetterboxAnchorTopLeft must match LetterboxAnchor::TOP_LEFT"); + +// MARK: - Private interfaces + +@interface ExecuTorchImageNormalization () +- (const Normalization &)nativeNormalization; +@end + +@interface ExecuTorchImageProcessorConfig () +- (ImageProcessorConfig)nativeConfig; +@end + +static ExecuTorchTensor *tensorFromResult( + executorch::runtime::Result &result, + NSError **error) { + if (!result.ok()) { + if (error) { + *error = ExecuTorchErrorWithCode((ExecuTorchErrorCode)result.error()); + } + return nil; + } + auto tensorPtr = std::move(result.get()); + // initWithNativeInstance moves out of tensorPtr, leaving it in a moved-from state. + return [[ExecuTorchTensor alloc] initWithNativeInstance:&tensorPtr]; +} + +// MARK: - ExecuTorchImageNormalization + +@implementation ExecuTorchImageNormalization { + Normalization _norm; +} + +- (instancetype)initWithNormalization:(Normalization)norm { + if (self = [super init]) { + _norm = norm; + } + return self; +} + ++ (instancetype)zeroToOne { + static ExecuTorchImageNormalization *instance = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + instance = [[self alloc] initWithNormalization:Normalization::zeroToOne()]; + }); + return instance; +} + ++ (instancetype)imagenet { + static ExecuTorchImageNormalization *instance = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + instance = [[self alloc] initWithNormalization:Normalization::imagenet()]; + }); + return instance; +} + +- (instancetype)initWithScaleFactor:(float)scaleFactor + mean:(NSArray *)mean + standardDeviation:(NSArray *)standardDeviation { + NSParameterAssert(mean.count == (NSUInteger)ImageProcessorConfig::kOutputChannels); + NSParameterAssert(standardDeviation.count == (NSUInteger)ImageProcessorConfig::kOutputChannels); + Normalization norm; + norm.scale_factor = scaleFactor; + for (NSUInteger i = 0; i < (NSUInteger)ImageProcessorConfig::kOutputChannels; ++i) { + norm.mean[i] = mean[i].floatValue; + norm.std_dev[i] = standardDeviation[i].floatValue; + } + // Reserved 4th (alpha) slot: identity so it stays divide-safe if a future + // path ever reads it (see Normalization in image_processor_config.h). + norm.mean[ImageProcessorConfig::kOutputChannels] = 0.0f; + norm.std_dev[ImageProcessorConfig::kOutputChannels] = 1.0f; + return [self initWithNormalization:norm]; +} + +- (const Normalization &)nativeNormalization { + return _norm; +} + +@end + +// MARK: - ExecuTorchImageProcessorConfig + +@implementation ExecuTorchImageProcessorConfig + +- (instancetype)initWithTargetWidth:(NSInteger)targetWidth + targetHeight:(NSInteger)targetHeight + resizeMode:(ExecuTorchImageResizeMode)resizeMode + letterboxAnchor:(ExecuTorchImageLetterboxAnchor)letterboxAnchor + padValue:(float)padValue + normalization:(ExecuTorchImageNormalization *)normalization + gpuMinInputPixels:(NSInteger)gpuMinInputPixels { + if (self = [super init]) { + _targetWidth = targetWidth; + _targetHeight = targetHeight; + _resizeMode = resizeMode; + _letterboxAnchor = letterboxAnchor; + _padValue = padValue; + _normalization = normalization; + _gpuMinInputPixels = gpuMinInputPixels; + } + return self; +} + +- (ImageProcessorConfig)nativeConfig { + ImageProcessorConfig config; + config.target_width = static_cast(_targetWidth); + config.target_height = static_cast(_targetHeight); + config.resize_mode = static_cast(_resizeMode); + config.letterbox_anchor = static_cast(_letterboxAnchor); + config.pad_value = _padValue; + config.normalization = [_normalization nativeNormalization]; + config.gpu_min_input_pixels = static_cast(_gpuMinInputPixels); + return config; +} + ++ (NSInteger)defaultGpuMinInputPixels { + return static_cast( + ImageProcessorConfig::kDefaultGpuMinInputPixels); +} + +@end + +// MARK: - ExecuTorchImageProcessor + +@implementation ExecuTorchImageProcessor { + std::optional _processor; +} + +- (instancetype)initWithConfig:(ExecuTorchImageProcessorConfig *)config { + NSParameterAssert(config); + if (self = [super init]) { + // Copy the config to avoid external mutations affecting processor.config + _config = [[ExecuTorchImageProcessorConfig alloc] + initWithTargetWidth:config.targetWidth + targetHeight:config.targetHeight + resizeMode:config.resizeMode + letterboxAnchor:config.letterboxAnchor + padValue:config.padValue + normalization:config.normalization + gpuMinInputPixels:config.gpuMinInputPixels]; + _processor.emplace([_config nativeConfig]); + } + return self; +} + +- (nullable ExecuTorchTensor *)processPixelBuffer:(_Nullable CVPixelBufferRef)pixelBuffer + error:(NSError **)error { + if (!pixelBuffer) { + if (error) { + *error = ExecuTorchErrorWithCode(ExecuTorchErrorCodeInvalidArgument); + } + return nil; + } + auto result = process_pixelbuffer(*_processor, pixelBuffer); + return tensorFromResult(result, error); +} + +- (BOOL)processPixelBuffer:(_Nullable CVPixelBufferRef)pixelBuffer + intoTensor:(ExecuTorchTensor *)tensor + error:(NSError **)error { + if (!pixelBuffer || !tensor) { + if (error) { + *error = ExecuTorchErrorWithCode(ExecuTorchErrorCodeInvalidArgument); + } + return NO; + } + auto* tensorPtr = reinterpret_cast(tensor.nativeInstance); + auto err = process_pixelbuffer_into( + *_processor, pixelBuffer, Orientation::UP, **tensorPtr); + if (err != executorch::runtime::Error::Ok) { + if (error) { + *error = ExecuTorchErrorWithCode((ExecuTorchErrorCode)err); + } + return NO; + } + return YES; +} + +- (ExecuTorchImageLetterboxPadding)computeLetterboxPaddingForInputWidth:(NSInteger)inputWidth + height:(NSInteger)inputHeight { + const auto padding = _processor->compute_letterbox_padding( + static_cast(inputWidth), static_cast(inputHeight)); + return {padding.first, padding.second}; +} + +@end diff --git a/extension/apple/ExecuTorch/__tests__/ImageProcessorTest.swift b/extension/apple/ExecuTorch/__tests__/ImageProcessorTest.swift new file mode 100644 index 00000000000..40cc7f941ed --- /dev/null +++ b/extension/apple/ExecuTorch/__tests__/ImageProcessorTest.swift @@ -0,0 +1,219 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +import CoreVideo +import ExecuTorch +import XCTest + +// These tests cover the ObjC/Swift binding layer only: config field forwarding, +// the CVPixelBuffer entry point, the reuse (process-into) path, the +// letterbox-padding bridge, and the nil guard. Image-processing correctness +// (color conversion, resize/letterbox math, normalization, CPU/GPU +// equivalence, format support) is owned by the C++ suite +// (extension/image/test/image_processor_test.cpp and +// image_processor_apple_test.cpp) and is intentionally not re-tested here. +class ImageProcessorTest: XCTestCase { + + // MARK: - Helper: Create BGRA CVPixelBuffer + + private func makeBGRAPixelBuffer(width: Int, height: Int, r: UInt8, g: UInt8, b: UInt8) -> CVPixelBuffer? { + var pixelBuffer: CVPixelBuffer? + let status = CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + nil, + &pixelBuffer + ) + guard status == kCVReturnSuccess, let buffer = pixelBuffer else { + return nil + } + + CVPixelBufferLockBaseAddress(buffer, []) + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + + if let base = CVPixelBufferGetBaseAddress(buffer) { + let stride = CVPixelBufferGetBytesPerRow(buffer) + let ptr = base.assumingMemoryBound(to: UInt8.self) + for row in 0.. C++ -> Tensor bridge end to end. + let config = ImageProcessorConfig(targetWidth: 4, targetHeight: 4) + let processor = ImageProcessor(config: config) + + guard let pixelBuffer = makeBGRAPixelBuffer(width: 8, height: 6, r: 200, g: 100, b: 50) else { + XCTFail("Failed to create BGRA pixel buffer") + return + } + + let output = Tensor.zeros(shape: [1, 3, 4, 4]) + try processor.process(pixelBuffer, into: output) + + let expected: Tensor = try processor.process(pixelBuffer) + XCTAssertEqual(output.shape, [1, 3, 4, 4]) + let outData = output.scalars() + let expData = expected.scalars() + XCTAssertEqual(outData.count, expData.count) + for i in 0.. must + // surface .invalidArgument through the binding; this is the binding-specific + // behavior the into: path exists for. + let config = ImageProcessorConfig(targetWidth: 4, targetHeight: 4) + let processor = ImageProcessor(config: config) + + guard let pixelBuffer = makeBGRAPixelBuffer(width: 8, height: 6, r: 200, g: 100, b: 50) else { + XCTFail("Failed to create BGRA pixel buffer") + return + } + + // Config expects [1, 3, 4, 4]; pass a mismatched output tensor. + let wrongShape = Tensor.zeros(shape: [1, 3, 8, 8]) + XCTAssertThrowsError(try processor.process(pixelBuffer, into: wrongShape)) { error in + let nsError = error as NSError + XCTAssertEqual(nsError.domain, ErrorDomain) + XCTAssertEqual(nsError.code, ErrorCode.invalidArgument.rawValue) + } + } + + // MARK: - Config round-trip tests + + func testConfigPropertyRoundTrip() throws { + // Construct config with non-default values and verify they round-trip + // through the processor. This catches dropped/misforwarded fields in + // initWithConfig and nativeConfig. + let config = ImageProcessorConfig( + targetWidth: 224, + targetHeight: 224, + resizeMode: .letterbox, + letterboxAnchor: .topLeft, + padValue: 0.5, + normalization: .imagenet(), + gpuMinInputPixels: ImageProcessorConfig.alwaysCPU + ) + let processor = ImageProcessor(config: config) + + // Verify all fields round-trip correctly + XCTAssertEqual(processor.config.targetWidth, 224) + XCTAssertEqual(processor.config.targetHeight, 224) + XCTAssertEqual(processor.config.resizeMode, .letterbox) + XCTAssertEqual(processor.config.letterboxAnchor, .topLeft) + XCTAssertEqual(processor.config.padValue, 0.5, accuracy: 1e-6) + XCTAssertEqual(processor.config.gpuMinInputPixels, ImageProcessorConfig.alwaysCPU) + // Normalization is a reference type, so we check it's the same instance + XCTAssertTrue(processor.config.normalization === config.normalization) + } + + func testDefaultInitializerUsesDefaultThreshold() throws { + // The convenience init inherits the C++ config's default gpuMinInputPixels. + let config = ImageProcessorConfig(targetWidth: 4, targetHeight: 4) + let processor = ImageProcessor(config: config) + + XCTAssertEqual( + processor.config.gpuMinInputPixels, + ImageProcessorConfig.defaultGpuMinInputPixels) + } + + // MARK: - Custom normalization + + func testCustomNormalizationApplied() throws { + // Verifies a custom ImageNormalization (scale/mean/std) actually flows + // through the binding into the C++ pipeline. zeroToOne yields pixel/255; + // with the same scale but mean 0.5 / std 0.5 the result is + // (pixel/255 - 0.5) / 0.5 == 2 * zeroToOne - 1, channel-wise. + guard let pixelBuffer = makeBGRAPixelBuffer(width: 8, height: 6, r: 200, g: 100, b: 50) else { + XCTFail("Failed to create BGRA pixel buffer") + return + } + + let baseConfig = ImageProcessorConfig(targetWidth: 4, targetHeight: 4) + let baseOutput = try ImageProcessor(config: baseConfig).process(pixelBuffer) + + let custom = ImageNormalization( + scaleFactor: 1.0 / 255.0, + mean: [0.5, 0.5, 0.5], + standardDeviation: [0.5, 0.5, 0.5]) + let customConfig = ImageProcessorConfig( + targetWidth: 4, + targetHeight: 4, + normalization: custom) + let customOutput = try ImageProcessor(config: customConfig).process(pixelBuffer) + + let base = baseOutput.scalars() + let got = customOutput.scalars() + XCTAssertEqual(base.count, got.count) + for i in 0..