Skip to content

Commit

Permalink
#91 yolo모델 오류 복구
Browse files Browse the repository at this point in the history
  • Loading branch information
@pknujsp
pknujsp committed Jun 3, 2023
1 parent 6320244 commit 7f803c2
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 54 deletions.
1 change: 1 addition & 0 deletions app/src/main/AndroidManifest.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

</intent-filter>
</activity>

</application>

</manifest>
18 changes: 6 additions & 12 deletions feature/camera/src/main/jni/ndkcamera.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@

#include "mat.h"

#include <future>
#include <vector>

static void onDisconnected(void *context, ACameraDevice *device) {
__android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device);
}
Expand All @@ -24,9 +21,10 @@ static void onImageAvailable(void *context, AImageReader *reader) {
AImage *image = 0;
media_status_t status = AImageReader_acquireLatestImage(reader, &image);

if (status != AMEDIA_OK)
if (status != AMEDIA_OK) {
// error
return;

}

int32_t format;
AImage_getFormat(image, &format);
Expand Down Expand Up @@ -61,8 +59,7 @@ static void onImageAvailable(void *context, AImageReader *reader) {
AImage_getPlaneData(image, 1, &u_data, &u_len);
AImage_getPlaneData(image, 2, &v_data, &v_len);

if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 &&
v_pixelStride == 2 &&
if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 && v_pixelStride == 2 &&
y_rowStride == width && u_rowStride == width && v_rowStride == width) {
// already nv21 :)
((NdkCamera *) context)->on_image((unsigned char *) y_data, (int) width, (int) height);
Expand Down Expand Up @@ -102,7 +99,6 @@ static void onImageAvailable(void *context, AImageReader *reader) {
}

AImage_delete(image);

}

static void onSessionActive(void *context, ACameraCaptureSession *session) {
Expand Down Expand Up @@ -150,7 +146,7 @@ NdkCamera::NdkCamera() {

// setup imagereader and its surface
{
AImageReader_new(640, 640, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/5, &image_reader);
AImageReader_new(640, 640, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/3, &image_reader);

AImageReader_ImageListener listener;
listener.context = this;
Expand Down Expand Up @@ -408,7 +404,6 @@ void NdkCameraWindow::set_window(ANativeWindow *_win) {
void NdkCameraWindow::on_image_render(cv::Mat &rgb) const {
}


void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv21_height) const {
// resolve orientation from camera_orientation and accelerometer_sensor
{
Expand Down Expand Up @@ -567,7 +562,6 @@ void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv
ANativeWindow_setBuffersGeometry(win, render_w, render_h, AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM);

ANativeWindow_Buffer buf;
// 기기 화면에 보여주기 위해 화면을 잠금
ANativeWindow_lock(win, &buf, NULL);

// scale to target size
Expand Down Expand Up @@ -602,6 +596,6 @@ void NdkCameraWindow::on_image(const unsigned char *nv21, int nv21_width, int nv
}
}
}
// 화면 잠금 풀고 처리한 데이터를 표시

ANativeWindow_unlockAndPost(win);
}
95 changes: 55 additions & 40 deletions feature/camera/src/main/jni/yolo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,9 @@

#include "cpu.h"

static const float prob_threshold = 0.45f;
static const float prob_threshold = 0.4f;
static const float nms_threshold = 0.45f;

static std::vector<Object> proposals;
static std::vector<GridAndStride> grid_strides;

static float fast_exp(float x) {
union {
uint32_t i;
Expand Down Expand Up @@ -66,29 +63,29 @@ static void qsort_descent_inplace(std::vector<Object> &faceobjects, int left, in
}
}

static void qsort_descent_inplace() {
if (proposals.empty())
static void qsort_descent_inplace(std::vector<Object> &faceobjects) {
if (faceobjects.empty())
return;

qsort_descent_inplace(proposals, 0, proposals.size() - 1);
qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1);
}

static void nms_sorted_bboxes(std::vector<int> &picked) {
static void nms_sorted_bboxes(const std::vector<Object> &faceobjects, std::vector<int> &picked) {
picked.clear();

const int n = proposals.size();
const int n = faceobjects.size();

std::vector<float> areas(n);
for (int i = 0; i < n; i++) {
areas[i] = proposals[i].rect.width * proposals[i].rect.height;
areas[i] = faceobjects[i].rect.width * faceobjects[i].rect.height;
}

for (int i = 0; i < n; i++) {
const Object &a = proposals[i];
const Object &a = faceobjects[i];

int keep = 1;
for (int j = 0; j < (int) picked.size(); j++) {
const Object &b = proposals[picked[j]];
const Object &b = faceobjects[picked[j]];

// intersection over union
float inter_area = intersection_area(a, b);
Expand All @@ -106,7 +103,7 @@ static void nms_sorted_bboxes(std::vector<int> &picked) {
static const std::vector<int> strides = {8, 16, 32};

static void
generate_grids_and_stride(const int target_w, const int target_h) {
generate_grids_and_stride(const int target_w, const int target_h, std::vector<GridAndStride> &grid_strides) {
int num_grid_w = 0;
int num_grid_h = 0;

Expand All @@ -127,11 +124,13 @@ generate_grids_and_stride(const int target_w, const int target_h) {
}
}

static void generate_proposals(const ncnn::Mat &pred) {
static void
generate_proposals(std::vector<GridAndStride> grid_strides, const ncnn::Mat &pred, std::vector<Object> &objects) {
const int num_points = grid_strides.size();
const int num_class = 1;
const int reg_max_1 = 16;

for (int i = 0; i < grid_strides.size(); i++) {
for (int i = 0; i < num_points; i++) {
const float *scores = pred.row(i) + 4 * reg_max_1;

// find label with max score
Expand Down Expand Up @@ -195,7 +194,7 @@ static void generate_proposals(const ncnn::Mat &pred) {
obj.label = label;
obj.prob = box_prob;

proposals.push_back(obj);
objects.push_back(obj);
}
}
}
Expand All @@ -217,9 +216,8 @@ Yolo::load(AAssetManager *mgr, const char *modeltype, int _target_size, const fl

yolo.opt = ncnn::Option();


#if NCNN_VULKAN
yolo.opt.use_vulkan_compute = true;
yolo.opt.use_vulkan_compute = use_gpu;
#endif

yolo.opt.num_threads = ncnn::get_cpu_count();
Expand All @@ -241,17 +239,11 @@ Yolo::load(AAssetManager *mgr, const char *modeltype, int _target_size, const fl
}


// sort objects by area
struct {
bool operator()(const Object &a, const Object &b) const {
return a.rect.area() > b.rect.area();
}
} objects_area_greater;

void Yolo::detect(const cv::Mat &rgb, std::vector<Object> &objects) {
int Yolo::detect(const cv::Mat &rgb, std::vector<Object> &objects) {
int width = rgb.cols;
int height = rgb.rows;

// pad to multiple of 32
int w = width;
int h = height;
float scale = 1.f;
Expand Down Expand Up @@ -279,47 +271,70 @@ void Yolo::detect(const cv::Mat &rgb, std::vector<Object> &objects) {

ex.input("images", in_pad);

proposals.clear();
std::vector<Object> proposals;

ncnn::Mat out;
ex.extract("output0", out);

// might have stride=64
generate_grids_and_stride(in_pad.w, in_pad.h);
// might have stride=64
std::vector<GridAndStride> grid_strides;
generate_grids_and_stride(in_pad.w, in_pad.h, grid_strides);

generate_proposals(out);
generate_proposals(grid_strides, out, proposals);

// sort all proposals by score from highest to lowest
qsort_descent_inplace();
qsort_descent_inplace(proposals);

// apply nms with nms_threshold
std::vector<int> picked;
nms_sorted_bboxes(picked);
nms_sorted_bboxes(proposals, picked);

int count = picked.size();

objects.resize(count);
for (int i = 0; i < count; i++) {
objects[i] = proposals[picked[i]];

float x0 = std::max(std::min((objects[i].rect.x - (wpad / 2)) / scale, (float) (width - 1)), 0.f);
float y0 = std::max(std::min((objects[i].rect.y - (hpad / 2)) / scale, (float) (height - 1)), 0.f);
float x1 = std::max(std::min((objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale, (float) (width - 1)), 0.f);
float y1 = std::max(std::min((objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale, (float) (height - 1)), 0.f);
// adjust offset to original unpadded
float x0 = (objects[i].rect.x - (wpad / 2)) / scale;
float y0 = (objects[i].rect.y - (hpad / 2)) / scale;
float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale;
float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale;

// clip
x0 = std::max(std::min(x0, (float) (width - 1)), 0.f);
y0 = std::max(std::min(y0, (float) (height - 1)), 0.f);
x1 = std::max(std::min(x1, (float) (width - 1)), 0.f);
y1 = std::max(std::min(y1, (float) (height - 1)), 0.f);

objects[i].rect.x = x0;
objects[i].rect.y = y0;
objects[i].rect.width = x1 - x0;
objects[i].rect.height = y1 - y0;
}

// std::sort(objects.begin(), objects.end(), objects_area_greater);
// sort objects by area
struct {
bool operator()(const Object &a, const Object &b) const {
return a.rect.area() > b.rect.area();
}
} objects_area_greater;
std::sort(objects.begin(), objects.end(), objects_area_greater);

return 0;
}

static const int color = 255;
static const cv::Scalar detectedRectColor(color, color, color);

void Yolo::draw(cv::Mat &rgb, const std::vector<Object> &objects) {
for (Object obj: objects) {
cv::rectangle(rgb, obj.rect, detectedRectColor, 2, cv::LINE_AA);

int Yolo::draw(cv::Mat &rgb, const std::vector<Object> &objects) {
if (objects.empty())
return 0;

for (const auto &obj: objects) {
cv::rectangle(rgb, obj.rect, detectedRectColor, 2);
}

return 0;
}
4 changes: 2 additions & 2 deletions feature/camera/src/main/jni/yolo.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ class Yolo {
int load(AAssetManager *mgr, const char *modeltype, int target_size, const float *mean_vals, const float *norm_vals, bool use_gpu =
true);

void detect(const cv::Mat &rgb, std::vector<Object> &objects);
int detect(const cv::Mat &rgb, std::vector<Object> &objects);

void draw(cv::Mat &rgb, const std::vector<Object> &objects);
int draw(cv::Mat &rgb, const std::vector<Object> &objects);

private:
ncnn::Net yolo;
Expand Down

0 comments on commit 7f803c2

Please sign in to comment.