diff --git a/feature/camera/src/main/jni/ndkcamera.cpp b/feature/camera/src/main/jni/ndkcamera.cpp index 90beea2d2..561e50ee7 100644 --- a/feature/camera/src/main/jni/ndkcamera.cpp +++ b/feature/camera/src/main/jni/ndkcamera.cpp @@ -9,6 +9,12 @@ #include "mat.h" +#include +#include + +static constexpr size_t num_tasks = 3; +static std::array, num_tasks> futures; + static void onDisconnected(void *context, ACameraDevice *device) { __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device); } @@ -18,87 +24,97 @@ static void onError(void *context, ACameraDevice *device, int error) { } static void onImageAvailable(void *context, AImageReader *reader) { - AImage *image = 0; - media_status_t status = AImageReader_acquireLatestImage(reader, &image); - - if (status != AMEDIA_OK) { - // error - return; - } + for (size_t i = 0; i < num_tasks; ++i) { + if (futures[i].wait_for(std::chrono::seconds(0)) != std::future_status::ready) { + return; + } - int32_t format; - AImage_getFormat(image, &format); - - - int32_t width = 0; - int32_t height = 0; - AImage_getWidth(image, &width); - AImage_getHeight(image, &height); - - int32_t y_pixelStride = 0; - int32_t u_pixelStride = 0; - int32_t v_pixelStride = 0; - AImage_getPlanePixelStride(image, 0, &y_pixelStride); - AImage_getPlanePixelStride(image, 1, &u_pixelStride); - AImage_getPlanePixelStride(image, 2, &v_pixelStride); - - int32_t y_rowStride = 0; - int32_t u_rowStride = 0; - int32_t v_rowStride = 0; - AImage_getPlaneRowStride(image, 0, &y_rowStride); - AImage_getPlaneRowStride(image, 1, &u_rowStride); - AImage_getPlaneRowStride(image, 2, &v_rowStride); - - uint8_t *y_data = 0; - uint8_t *u_data = 0; - uint8_t *v_data = 0; - int y_len = 0; - int u_len = 0; - int v_len = 0; - AImage_getPlaneData(image, 0, &y_data, &y_len); - AImage_getPlaneData(image, 1, &u_data, &u_len); - AImage_getPlaneData(image, 2, &v_data, &v_len); - - if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 && v_pixelStride == 2 && - y_rowStride == width && u_rowStride == width && v_rowStride == width) { - // already nv21 :) - ((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height); - } else { - // construct nv21 - unsigned char *nv21 = new unsigned char[width * height + width * height / 2]; - { - // Y - unsigned char *yptr = nv21; - for (int y = 0; y < height; y++) { - const unsigned char *y_data_ptr = y_data + y_rowStride * y; - for (int x = 0; x < width; x++) { - yptr[0] = y_data_ptr[0]; - yptr++; - y_data_ptr += y_pixelStride; + futures[i] = std::async(std::launch::async, [&reader, &context] { + AImage *image = 0; + media_status_t status = AImageReader_acquireLatestImage(reader, &image); + + if (status != AMEDIA_OK) + return; + + + int32_t format; + AImage_getFormat(image, &format); + + + int32_t width = 0; + int32_t height = 0; + AImage_getWidth(image, &width); + AImage_getHeight(image, &height); + + int32_t y_pixelStride = 0; + int32_t u_pixelStride = 0; + int32_t v_pixelStride = 0; + AImage_getPlanePixelStride(image, 0, &y_pixelStride); + AImage_getPlanePixelStride(image, 1, &u_pixelStride); + AImage_getPlanePixelStride(image, 2, &v_pixelStride); + + int32_t y_rowStride = 0; + int32_t u_rowStride = 0; + int32_t v_rowStride = 0; + AImage_getPlaneRowStride(image, 0, &y_rowStride); + AImage_getPlaneRowStride(image, 1, &u_rowStride); + AImage_getPlaneRowStride(image, 2, &v_rowStride); + + uint8_t *y_data = 0; + uint8_t *u_data = 0; + uint8_t *v_data = 0; + int y_len = 0; + int u_len = 0; + int v_len = 0; + AImage_getPlaneData(image, 0, &y_data, &y_len); + AImage_getPlaneData(image, 1, &u_data, &u_len); + AImage_getPlaneData(image, 2, &v_data, &v_len); + + if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 && + v_pixelStride == 2 && + y_rowStride == width && u_rowStride == width && v_rowStride == width) { + // already nv21 :) + ((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height); + } else { + // construct nv21 + unsigned char *nv21 = new unsigned char[width * height + width * height / 2]; + { + // Y + unsigned char *yptr = nv21; + for (int y = 0; y < height; y++) { + const unsigned char *y_data_ptr = y_data + y_rowStride * y; + for (int x = 0; x < width; x++) { + yptr[0] = y_data_ptr[0]; + yptr++; + y_data_ptr += y_pixelStride; + } + } + + // UV + unsigned char *uvptr = nv21 + width * height; + for (int y = 0; y < height / 2; y++) { + const unsigned char *v_data_ptr = v_data + v_rowStride * y; + const unsigned char *u_data_ptr = u_data + u_rowStride * y; + for (int x = 0; x < width / 2; x++) { + uvptr[0] = v_data_ptr[0]; + uvptr[1] = u_data_ptr[0]; + uvptr += 2; + v_data_ptr += v_pixelStride; + u_data_ptr += u_pixelStride; + } + } } - } - // UV - unsigned char *uvptr = nv21 + width * height; - for (int y = 0; y < height / 2; y++) { - const unsigned char *v_data_ptr = v_data + v_rowStride * y; - const unsigned char *u_data_ptr = u_data + u_rowStride * y; - for (int x = 0; x < width / 2; x++) { - uvptr[0] = v_data_ptr[0]; - uvptr[1] = u_data_ptr[0]; - uvptr += 2; - v_data_ptr += v_pixelStride; - u_data_ptr += u_pixelStride; - } - } - } + ((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height); - ((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height); + delete[] nv21; + } - delete[] nv21; + AImage_delete(image); + }); } - AImage_delete(image); + } static void onSessionActive(void *context, ACameraCaptureSession *session) { @@ -146,7 +162,7 @@ NdkCamera::NdkCamera() { // setup imagereader and its surface { - AImageReader_new(640, 640, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/3, &image_reader); + AImageReader_new(640, 640, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/5, &image_reader); AImageReader_ImageListener listener; listener.context = this; @@ -562,6 +578,7 @@ void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv ANativeWindow_setBuffersGeometry(win, render_w, render_h, AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM); ANativeWindow_Buffer buf; + // 기기 화면에 보여주기 위해 화면을 잠금 ANativeWindow_lock(win, &buf, NULL); // scale to target size @@ -596,6 +613,6 @@ void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv } } } - + // 화면 잠금 풀고 처리한 데이터를 표시 ANativeWindow_unlockAndPost(win); } \ No newline at end of file diff --git a/feature/camera/src/main/jni/yolo.cpp b/feature/camera/src/main/jni/yolo.cpp index 0d7a73b89..88752a5ef 100644 --- a/feature/camera/src/main/jni/yolo.cpp +++ b/feature/camera/src/main/jni/yolo.cpp @@ -238,8 +238,14 @@ Yolo::load(AAssetManager *mgr, const char *modeltype, int _target_size, const fl return 0; } +// sort objects by area +struct { + bool operator()(const Object &a, const Object &b) const { + return a.rect.area() > b.rect.area(); + } +} objects_area_greater; -int Yolo::detect(const cv::Mat &rgb, std::vector &objects) { +void Yolo::detect(const cv::Mat &rgb, std::vector &objects) { int width = rgb.cols; int height = rgb.rows; @@ -295,17 +301,10 @@ int Yolo::detect(const cv::Mat &rgb, std::vector &objects) { for (int i = 0; i < count; i++) { objects[i] = proposals[picked[i]]; - // adjust offset to original unpadded - float x0 = (objects[i].rect.x - (wpad / 2)) / scale; - float y0 = (objects[i].rect.y - (hpad / 2)) / scale; - float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; - float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; - - // clip - x0 = std::max(std::min(x0, (float) (width - 1)), 0.f); - y0 = std::max(std::min(y0, (float) (height - 1)), 0.f); - x1 = std::max(std::min(x1, (float) (width - 1)), 0.f); - y1 = std::max(std::min(y1, (float) (height - 1)), 0.f); + float x0 = std::max(std::min((objects[i].rect.x - (wpad / 2)) / scale, (float) (width - 1)), 0.f); + float y0 = std::max(std::min((objects[i].rect.y - (hpad / 2)) / scale, (float) (height - 1)), 0.f); + float x1 = std::max(std::min((objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale, (float) (width - 1)), 0.f); + float y1 = std::max(std::min((objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale, (float) (height - 1)), 0.f); objects[i].rect.x = x0; objects[i].rect.y = y0; @@ -313,28 +312,14 @@ int Yolo::detect(const cv::Mat &rgb, std::vector &objects) { objects[i].rect.height = y1 - y0; } - // sort objects by area - struct { - bool operator()(const Object &a, const Object &b) const { - return a.rect.area() > b.rect.area(); - } - } objects_area_greater; std::sort(objects.begin(), objects.end(), objects_area_greater); - - return 0; } static const int color = 255; static const cv::Scalar detectedRectColor(color, color, color); - -int Yolo::draw(cv::Mat &rgb, const std::vector &objects) { - if (objects.empty()) - return 0; - - for (const auto &obj: objects) { - cv::rectangle(rgb, obj.rect, detectedRectColor, 2); +void Yolo::draw(cv::Mat &rgb, const std::vector &objects) { + for (Object obj: objects) { + cv::rectangle(rgb, obj.rect, detectedRectColor, 2, cv::LINE_AA); } - - return 0; } \ No newline at end of file diff --git a/feature/camera/src/main/jni/yolo.h b/feature/camera/src/main/jni/yolo.h index ab51e7e61..cebd8b505 100644 --- a/feature/camera/src/main/jni/yolo.h +++ b/feature/camera/src/main/jni/yolo.h @@ -27,9 +27,9 @@ class Yolo { int load(AAssetManager *mgr, const char *modeltype, int target_size, const float *mean_vals, const float *norm_vals, bool use_gpu = true); - int detect(const cv::Mat &rgb, std::vector &objects); + void detect(const cv::Mat &rgb, std::vector &objects); - int draw(cv::Mat &rgb, const std::vector &objects); + void draw(cv::Mat &rgb, const std::vector &objects); private: ncnn::Net yolo; diff --git a/feature/camera/src/main/jni/yolov8ncnn.cpp b/feature/camera/src/main/jni/yolov8ncnn.cpp index a0e490787..be6b4c2d9 100644 --- a/feature/camera/src/main/jni/yolov8ncnn.cpp +++ b/feature/camera/src/main/jni/yolov8ncnn.cpp @@ -159,15 +159,17 @@ static std::vector objs; void MyNdkCamera::on_image_render(cv::Mat &rgb) const { { - ncnn::MutexLockGuard g(lock); + //ncnn::MutexLockGuard g(lock); if (g_yolo) { objs.clear(); g_yolo->detect(rgb, objs); g_yolo->draw(rgb, objs); - currentRgb = &rgb; - detectedObjects = objs; + if (!objs.empty()) { + currentRgb = &rgb; + detectedObjects = objs; + } } }