Skip to content

Commit

Permalink
#91 카메라 처리 로직 최적화
Browse files Browse the repository at this point in the history
  • Loading branch information
@pknujsp
pknujsp committed Jun 2, 2023
1 parent dad2251 commit c86fa9e
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 110 deletions.
169 changes: 93 additions & 76 deletions feature/camera/src/main/jni/ndkcamera.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@

#include "mat.h"

#include <future>
#include <vector>

static constexpr size_t num_tasks = 3;
static std::array<std::future<void>, num_tasks> futures;

static void onDisconnected(void *context, ACameraDevice *device) {
__android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device);
}
Expand All @@ -18,87 +24,97 @@ static void onError(void *context, ACameraDevice *device, int error) {
}

static void onImageAvailable(void *context, AImageReader *reader) {
AImage *image = 0;
media_status_t status = AImageReader_acquireLatestImage(reader, &image);

if (status != AMEDIA_OK) {
// error
return;
}
for (size_t i = 0; i < num_tasks; ++i) {
if (futures[i].wait_for(std::chrono::seconds(0)) != std::future_status::ready) {
return;
}

int32_t format;
AImage_getFormat(image, &format);


int32_t width = 0;
int32_t height = 0;
AImage_getWidth(image, &width);
AImage_getHeight(image, &height);

int32_t y_pixelStride = 0;
int32_t u_pixelStride = 0;
int32_t v_pixelStride = 0;
AImage_getPlanePixelStride(image, 0, &y_pixelStride);
AImage_getPlanePixelStride(image, 1, &u_pixelStride);
AImage_getPlanePixelStride(image, 2, &v_pixelStride);

int32_t y_rowStride = 0;
int32_t u_rowStride = 0;
int32_t v_rowStride = 0;
AImage_getPlaneRowStride(image, 0, &y_rowStride);
AImage_getPlaneRowStride(image, 1, &u_rowStride);
AImage_getPlaneRowStride(image, 2, &v_rowStride);

uint8_t *y_data = 0;
uint8_t *u_data = 0;
uint8_t *v_data = 0;
int y_len = 0;
int u_len = 0;
int v_len = 0;
AImage_getPlaneData(image, 0, &y_data, &y_len);
AImage_getPlaneData(image, 1, &u_data, &u_len);
AImage_getPlaneData(image, 2, &v_data, &v_len);

if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 && v_pixelStride == 2 &&
y_rowStride == width && u_rowStride == width && v_rowStride == width) {
// already nv21 :)
((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height);
} else {
// construct nv21
unsigned char *nv21 = new unsigned char[width * height + width * height / 2];
{
// Y
unsigned char *yptr = nv21;
for (int y = 0; y < height; y++) {
const unsigned char *y_data_ptr = y_data + y_rowStride * y;
for (int x = 0; x < width; x++) {
yptr[0] = y_data_ptr[0];
yptr++;
y_data_ptr += y_pixelStride;
futures[i] = std::async(std::launch::async, [&reader, &context] {
AImage *image = 0;
media_status_t status = AImageReader_acquireLatestImage(reader, &image);

if (status != AMEDIA_OK)
return;


int32_t format;
AImage_getFormat(image, &format);


int32_t width = 0;
int32_t height = 0;
AImage_getWidth(image, &width);
AImage_getHeight(image, &height);

int32_t y_pixelStride = 0;
int32_t u_pixelStride = 0;
int32_t v_pixelStride = 0;
AImage_getPlanePixelStride(image, 0, &y_pixelStride);
AImage_getPlanePixelStride(image, 1, &u_pixelStride);
AImage_getPlanePixelStride(image, 2, &v_pixelStride);

int32_t y_rowStride = 0;
int32_t u_rowStride = 0;
int32_t v_rowStride = 0;
AImage_getPlaneRowStride(image, 0, &y_rowStride);
AImage_getPlaneRowStride(image, 1, &u_rowStride);
AImage_getPlaneRowStride(image, 2, &v_rowStride);

uint8_t *y_data = 0;
uint8_t *u_data = 0;
uint8_t *v_data = 0;
int y_len = 0;
int u_len = 0;
int v_len = 0;
AImage_getPlaneData(image, 0, &y_data, &y_len);
AImage_getPlaneData(image, 1, &u_data, &u_len);
AImage_getPlaneData(image, 2, &v_data, &v_len);

if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 &&
v_pixelStride == 2 &&
y_rowStride == width && u_rowStride == width && v_rowStride == width) {
// already nv21 :)
((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height);
} else {
// construct nv21
unsigned char *nv21 = new unsigned char[width * height + width * height / 2];
{
// Y
unsigned char *yptr = nv21;
for (int y = 0; y < height; y++) {
const unsigned char *y_data_ptr = y_data + y_rowStride * y;
for (int x = 0; x < width; x++) {
yptr[0] = y_data_ptr[0];
yptr++;
y_data_ptr += y_pixelStride;
}
}

// UV
unsigned char *uvptr = nv21 + width * height;
for (int y = 0; y < height / 2; y++) {
const unsigned char *v_data_ptr = v_data + v_rowStride * y;
const unsigned char *u_data_ptr = u_data + u_rowStride * y;
for (int x = 0; x < width / 2; x++) {
uvptr[0] = v_data_ptr[0];
uvptr[1] = u_data_ptr[0];
uvptr += 2;
v_data_ptr += v_pixelStride;
u_data_ptr += u_pixelStride;
}
}
}
}

// UV
unsigned char *uvptr = nv21 + width * height;
for (int y = 0; y < height / 2; y++) {
const unsigned char *v_data_ptr = v_data + v_rowStride * y;
const unsigned char *u_data_ptr = u_data + u_rowStride * y;
for (int x = 0; x < width / 2; x++) {
uvptr[0] = v_data_ptr[0];
uvptr[1] = u_data_ptr[0];
uvptr += 2;
v_data_ptr += v_pixelStride;
u_data_ptr += u_pixelStride;
}
}
}
((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height);

((NdkCamera *) context)->on_image((unsigned char *) nv21, (int) width, (int) height);
delete[] nv21;
}

delete[] nv21;
AImage_delete(image);
});
}

AImage_delete(image);

}

static void onSessionActive(void *context, ACameraCaptureSession *session) {
Expand Down Expand Up @@ -146,7 +162,7 @@ NdkCamera::NdkCamera() {

// setup imagereader and its surface
{
AImageReader_new(640, 640, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/3, &image_reader);
AImageReader_new(640, 640, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/5, &image_reader);

AImageReader_ImageListener listener;
listener.context = this;
Expand Down Expand Up @@ -562,6 +578,7 @@ void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv
ANativeWindow_setBuffersGeometry(win, render_w, render_h, AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM);

ANativeWindow_Buffer buf;
// 기기 화면에 보여주기 위해 화면을 잠금
ANativeWindow_lock(win, &buf, NULL);

// scale to target size
Expand Down Expand Up @@ -596,6 +613,6 @@ void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv
}
}
}

// 화면 잠금 풀고 처리한 데이터를 표시
ANativeWindow_unlockAndPost(win);
}
43 changes: 14 additions & 29 deletions feature/camera/src/main/jni/yolo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,14 @@ Yolo::load(AAssetManager *mgr, const char *modeltype, int _target_size, const fl
return 0;
}

// sort objects by area
struct {
bool operator()(const Object &a, const Object &b) const {
return a.rect.area() > b.rect.area();
}
} objects_area_greater;

int Yolo::detect(const cv::Mat &rgb, std::vector<Object> &objects) {
void Yolo::detect(const cv::Mat &rgb, std::vector<Object> &objects) {
int width = rgb.cols;
int height = rgb.rows;

Expand Down Expand Up @@ -295,46 +301,25 @@ int Yolo::detect(const cv::Mat &rgb, std::vector<Object> &objects) {
for (int i = 0; i < count; i++) {
objects[i] = proposals[picked[i]];

// adjust offset to original unpadded
float x0 = (objects[i].rect.x - (wpad / 2)) / scale;
float y0 = (objects[i].rect.y - (hpad / 2)) / scale;
float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale;
float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale;

// clip
x0 = std::max(std::min(x0, (float) (width - 1)), 0.f);
y0 = std::max(std::min(y0, (float) (height - 1)), 0.f);
x1 = std::max(std::min(x1, (float) (width - 1)), 0.f);
y1 = std::max(std::min(y1, (float) (height - 1)), 0.f);
float x0 = std::max(std::min((objects[i].rect.x - (wpad / 2)) / scale, (float) (width - 1)), 0.f);
float y0 = std::max(std::min((objects[i].rect.y - (hpad / 2)) / scale, (float) (height - 1)), 0.f);
float x1 = std::max(std::min((objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale, (float) (width - 1)), 0.f);
float y1 = std::max(std::min((objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale, (float) (height - 1)), 0.f);

objects[i].rect.x = x0;
objects[i].rect.y = y0;
objects[i].rect.width = x1 - x0;
objects[i].rect.height = y1 - y0;
}

// sort objects by area
struct {
bool operator()(const Object &a, const Object &b) const {
return a.rect.area() > b.rect.area();
}
} objects_area_greater;
std::sort(objects.begin(), objects.end(), objects_area_greater);

return 0;
}

static const int color = 255;
static const cv::Scalar detectedRectColor(color, color, color);


int Yolo::draw(cv::Mat &rgb, const std::vector<Object> &objects) {
if (objects.empty())
return 0;

for (const auto &obj: objects) {
cv::rectangle(rgb, obj.rect, detectedRectColor, 2);
void Yolo::draw(cv::Mat &rgb, const std::vector<Object> &objects) {
for (Object obj: objects) {
cv::rectangle(rgb, obj.rect, detectedRectColor, 2, cv::LINE_AA);
}

return 0;
}
4 changes: 2 additions & 2 deletions feature/camera/src/main/jni/yolo.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ class Yolo {
int load(AAssetManager *mgr, const char *modeltype, int target_size, const float *mean_vals, const float *norm_vals, bool use_gpu =
true);

int detect(const cv::Mat &rgb, std::vector<Object> &objects);
void detect(const cv::Mat &rgb, std::vector<Object> &objects);

int draw(cv::Mat &rgb, const std::vector<Object> &objects);
void draw(cv::Mat &rgb, const std::vector<Object> &objects);

private:
ncnn::Net yolo;
Expand Down
8 changes: 5 additions & 3 deletions feature/camera/src/main/jni/yolov8ncnn.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,15 +159,17 @@ static std::vector<Object> objs;

void MyNdkCamera::on_image_render(cv::Mat &rgb) const {
{
ncnn::MutexLockGuard g(lock);
//ncnn::MutexLockGuard g(lock);

if (g_yolo) {
objs.clear();
g_yolo->detect(rgb, objs);
g_yolo->draw(rgb, objs);

currentRgb = &rgb;
detectedObjects = objs;
if (!objs.empty()) {
currentRgb = &rgb;
detectedObjects = objs;
}
}
}

Expand Down

0 comments on commit c86fa9e

Please sign in to comment.