diff --git a/Dockerfile b/Dockerfile index ef1af12..1e781dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,16 @@ -FROM python:3.9 - -WORKDIR /app -COPY ./requirements.txt . -RUN pip install --upgrade pip -RUN pip install -U wheel cmake -RUN apt-get update && apt-get install -y python3-opencv -RUN pip install opencv-python -RUN pip install -r requirements.txt - -COPY . . - -EXPOSE 5005 -USER 1001 - -CMD ["python", "./face_rec_code/app.py"] +FROM python:3.9 + +WORKDIR /app +COPY ./requirements.txt . +RUN pip install --upgrade pip +RUN pip install -U wheel cmake +RUN apt-get update && apt-get install -y python3-opencv +RUN pip install opencv-python +RUN pip install -r requirements.txt + +COPY . . + +EXPOSE 5005 +USER 1001 + +CMD ["python", "./face_rec_code/app.py"] diff --git a/changelog.md b/changelog.md index 1ad4716..4006f53 100644 --- a/changelog.md +++ b/changelog.md @@ -1,26 +1,26 @@ -# Changelog - -## 25-Sep-22 - git:feature-start_end -1. added `video_face_rec.py/faces_split_timestamps` return value for all the enteries a faces appears -2. added `video_face_rec.py/refine_results` to remove single frames faces from enteries return, but not activated in 1st appearance return -3. fixed `timestamp` = 0 by the end of the video -4. fixed `target_fps` having value less than 1 which resulted in corrupted video output - -## 24-Sep-22 - git:feature-custm_analysis -1. restructured the files into folders - - `/face_rec_code/`: for all the python code - - `/face_rec_files/`: for the videos and pictures -2. split the code into 2 python files - - `face_rec_code/video_face_rec.py` for loading the folder and files then the analysis code - - `face_rec_code/app.py` for running the flask server -3. output unknown persons photos and labeled video are saved inside a folder by the name of the video inside `./unknown/` -4. changed `video_face_rec.py/run` to `video_face_rec.py/pipeline` with default parameters in case no parameters passed -5. added 2 `app.py/run` methods in the server - - `app.py/run_defaults` - GET: uses the `video_face_rec.py/pipeline` with default parameters, doesn't take any arguments - - `app.py/run_custom` - POST: uses the `video_face_rec.py/pipeline` with customs parameters, takes the parameters from json object from the post request -6. enhanced the `video_face_rec.py/confirm_dirs` to check all the directories and files are correct -7. added `video_face_rec.py/fetch_video_full_dir` to get the video for the analysis (either from default or custom run) -8. added tolerance parameter in the analysis -9. moved code for saving photos and video to seperate functions `video_face_rec.py/save_photo` & `video_face_rec.py/save_video` -10. changed the flask server running to more stable `app.py/WSGIServer` +# Changelog + +## 25-Sep-22 - git:feature-start_end +1. added `video_face_rec.py/faces_split_timestamps` return value for all the enteries a faces appears +2. added `video_face_rec.py/refine_results` to remove single frames faces from enteries return, but not activated in 1st appearance return +3. fixed `timestamp` = 0 by the end of the video +4. fixed `target_fps` having value less than 1 which resulted in corrupted video output + +## 24-Sep-22 - git:feature-custm_analysis +1. restructured the files into folders + - `/face_rec_code/`: for all the python code + - `/face_rec_files/`: for the videos and pictures +2. split the code into 2 python files + - `face_rec_code/video_face_rec.py` for loading the folder and files then the analysis code + - `face_rec_code/app.py` for running the flask server +3. output unknown persons photos and labeled video are saved inside a folder by the name of the video inside `./unknown/` +4. changed `video_face_rec.py/run` to `video_face_rec.py/pipeline` with default parameters in case no parameters passed +5. added 2 `app.py/run` methods in the server + - `app.py/run_defaults` - GET: uses the `video_face_rec.py/pipeline` with default parameters, doesn't take any arguments + - `app.py/run_custom` - POST: uses the `video_face_rec.py/pipeline` with customs parameters, takes the parameters from json object from the post request +6. enhanced the `video_face_rec.py/confirm_dirs` to check all the directories and files are correct +7. added `video_face_rec.py/fetch_video_full_dir` to get the video for the analysis (either from default or custom run) +8. added tolerance parameter in the analysis +9. moved code for saving photos and video to seperate functions `video_face_rec.py/save_photo` & `video_face_rec.py/save_video` +10. changed the flask server running to more stable `app.py/WSGIServer` 11. analysis terminal prints are now in minutes & seconds, ex:"time: 0m:25s - Found 1 faces, recognized: Unknown" \ No newline at end of file diff --git a/face_rec_code/app.py b/face_rec_code/app.py index 57dd622..44efb57 100644 --- a/face_rec_code/app.py +++ b/face_rec_code/app.py @@ -1,104 +1,104 @@ -import os -from flask import Flask, request, jsonify -import dlib -from gevent.pywsgi import WSGIServer -from video_face_rec import confirm_dirs, pipeline - -import warnings -warnings.simplefilter("ignore") - -app = Flask(__name__) -app.config['JSON_SORT_KEYS'] = False -APP_DIR = os.path.dirname(__file__) -DOCKER_DIR = os.path.dirname(APP_DIR) - - -@app.route("/healthcheck", methods=["GET"]) -def healthcheck(): - status, _, _, _, _, msg = confirm_dirs(DOCKER_DIR) - if status: - status_code = 200 - else: - status_code = 425 # error in healthcheck - obj = {"status": msg} - return jsonify(obj), status_code - -@app.route("/run_defaults", methods=["GET"]) -def run_defaults(): - print("-----default analysis running") - obj, status_code = pipeline() - return jsonify(obj), status_code - -@app.route("/run_custom", methods=["POST"]) -def run_custom(): - print("-----custom analysis running") - - try: - file_name = request.json["file_name"] - print("-----file name found in request: ", file_name) - except: - file_name = None - print("-----file name not found in request") - - try: - model = request.json["model"] - print("-----model found in request: ", model) - except: - model = 'hog' - print("-----model name not found in request") - - try: - skip_frames = request.json["skip_frames"] - print("-----skip_frames found in request: ", skip_frames) - except: - skip_frames = 5 - print("-----skip_frames not found in request") - - try: - resiz_factor = request.json["resiz_factor"] - print("-----resiz_factor found in request: ", resiz_factor) - except: - resiz_factor = 1 - print("-----resiz_factor not found in request") - - try: - n_upscale = request.json["n_upscale"] - print("-----n_upscale found in request: ", n_upscale) - except: - n_upscale = 1 - print("-----n_upscale not found in request") - - try: - num_jitters = request.json["num_jitters"] - print("-----num_jitters found in request: ", num_jitters) - except: - num_jitters = 1 - print("-----num_jitters not found in request") - - try: - tolerance = request.json["tolerance"] - print("-----tolerance found in request: ", tolerance) - except: - tolerance = 0.6 - print("-----tolerance not found in request") - - obj, status_code = pipeline(video_name=file_name, model=model, skip_frames=skip_frames, resiz_factor=resiz_factor, n_upscale=n_upscale, num_jitters=num_jitters, tolerance=tolerance) - return jsonify(obj), status_code - - - -if __name__ == "__main__": - print("-----working on: ",DOCKER_DIR) - try: - cuda = dlib.DLIB_USE_CUDA - if cuda: - print("-----running on GPU") - else: - print("-----running on CPU") - except: - print("-----couldnn't detect CUDA") - - # app.run(debug=True, host="0.0.0.0", port="3000") - - http_server = WSGIServer(('', 3000), app) - http_server.serve_forever() +import os +from flask import Flask, request, jsonify +import dlib +from gevent.pywsgi import WSGIServer +from video_face_rec import confirm_dirs, pipeline + +import warnings +warnings.simplefilter("ignore") + +app = Flask(__name__) +app.config['JSON_SORT_KEYS'] = False +APP_DIR = os.path.dirname(__file__) +DOCKER_DIR = os.path.dirname(APP_DIR) + + +@app.route("/healthcheck", methods=["GET"]) +def healthcheck(): + status, _, _, _, _, msg = confirm_dirs(DOCKER_DIR) + if status: + status_code = 200 + else: + status_code = 425 # error in healthcheck + obj = {"status": msg} + return jsonify(obj), status_code + +@app.route("/run_defaults", methods=["GET"]) +def run_defaults(): + print("-----default analysis running") + obj, status_code = pipeline() + return jsonify(obj), status_code + +@app.route("/run_custom", methods=["POST"]) +def run_custom(): + print("-----custom analysis running") + + try: + file_name = request.json["file_name"] + print("-----file name found in request: ", file_name) + except: + file_name = None + print("-----file name not found in request") + + try: + model = request.json["model"] + print("-----model found in request: ", model) + except: + model = 'hog' + print("-----model name not found in request") + + try: + skip_frames = request.json["skip_frames"] + print("-----skip_frames found in request: ", skip_frames) + except: + skip_frames = 5 + print("-----skip_frames not found in request") + + try: + resiz_factor = request.json["resiz_factor"] + print("-----resiz_factor found in request: ", resiz_factor) + except: + resiz_factor = 1 + print("-----resiz_factor not found in request") + + try: + n_upscale = request.json["n_upscale"] + print("-----n_upscale found in request: ", n_upscale) + except: + n_upscale = 1 + print("-----n_upscale not found in request") + + try: + num_jitters = request.json["num_jitters"] + print("-----num_jitters found in request: ", num_jitters) + except: + num_jitters = 1 + print("-----num_jitters not found in request") + + try: + tolerance = request.json["tolerance"] + print("-----tolerance found in request: ", tolerance) + except: + tolerance = 0.6 + print("-----tolerance not found in request") + + obj, status_code = pipeline(video_name=file_name, model=model, skip_frames=skip_frames, resiz_factor=resiz_factor, n_upscale=n_upscale, num_jitters=num_jitters, tolerance=tolerance) + return jsonify(obj), status_code + + + +if __name__ == "__main__": + print("-----working on: ",DOCKER_DIR) + try: + cuda = dlib.DLIB_USE_CUDA + if cuda: + print("-----running on GPU") + else: + print("-----running on CPU") + except: + print("-----couldnn't detect CUDA") + + # app.run(debug=True, host="0.0.0.0", port="3000") + + http_server = WSGIServer(('', 3000), app) + http_server.serve_forever() diff --git a/face_rec_code/request_test.py b/face_rec_code/request_test.py index d6d2c5e..31b3cd8 100644 --- a/face_rec_code/request_test.py +++ b/face_rec_code/request_test.py @@ -1,21 +1,21 @@ -import requests - -# test the server requests -port = "3000" -post_obj = { - "file_name": "3.mp4", - # "model": "hog", - "skip_frames": 30, - # "resiz_factor": 1, - # "num_jitters": 1, - # "tolerance": 0.6, - } -# method = "run_defaults" -method = "run_custom" -host = f'http://localhost:{port}/{method}' -r = requests.post(host, json=post_obj) -# r = requests.get(host) - - -print(r.status_code) -print(r.text) +import requests + +# test the server requests +port = "3000" +post_obj = { + "file_name": "3.mp4", + # "model": "hog", + "skip_frames": 30, + # "resiz_factor": 1, + # "num_jitters": 1, + # "tolerance": 0.6, + } +# method = "run_defaults" +method = "run_custom" +host = f'http://localhost:{port}/{method}' +r = requests.post(host, json=post_obj) +# r = requests.get(host) + + +print(r.status_code) +print(r.text) diff --git a/face_rec_code/video_face_rec.py b/face_rec_code/video_face_rec.py index fcb5d57..79b2055 100644 --- a/face_rec_code/video_face_rec.py +++ b/face_rec_code/video_face_rec.py @@ -1,384 +1,384 @@ -import face_recognition -import os -import cv2 -import numpy as np -import dlib -# import time - -APP_DIR = os.path.dirname(__file__) -DOCKER_DIR = os.path.dirname(APP_DIR) - -# confirm required dirs exist and append the current dir to the required folders name to produce absolute paths -def confirm_dirs(parent_dir, video_name): - mount_relative_dir = 'face_rec_files' - know_faces_relative_dir = 'known' - unknown_faces_relative_dir = 'unknown' - video_relative_dir = "feed" - - mount_full_dir = os.path.normpath(os.path.join(parent_dir, mount_relative_dir)) - mount_exist = os.path.isdir(mount_full_dir) - - status_msg = "" - - #if error in mount folder, exit confirm_dirs and return status False - if not(mount_exist): - status_msg = "root working directory not found" - print(status_msg) - return False, "", "", "", "", status_msg - - # load all directories and get full paths - try: - know_faces_full_dir = os.path.normpath(os.path.join(mount_full_dir, know_faces_relative_dir)) - unknown_faces_full_dir = os.path.normpath(os.path.join(mount_full_dir, unknown_faces_relative_dir)) - video_folder_full_dir = os.path.normpath(os.path.join(mount_full_dir, video_relative_dir)) - - know_faces_full_dir_exist = os.path.isdir(know_faces_full_dir) - unknown_faces_full_dir_exist = os.path.isdir(unknown_faces_full_dir) - video_folder_full_dir_exist = os.path.isdir(video_folder_full_dir) - all_dir_exist = know_faces_full_dir_exist and unknown_faces_full_dir_exist and video_folder_full_dir_exist - except Exception as e: - print("-----error11: ", e) - status_msg = "unexpected error, please check all directories and videos" - return False, "", "", "", "", status_msg - - #if error in directories structure, exit confirm_dirs and return status False - if not(all_dir_exist): - status_msg = f"Error in dirs - known: {know_faces_full_dir_exist}, unknown: {unknown_faces_full_dir_exist}, video: {video_folder_full_dir_exist}" - print(status_msg) - return False, "", "", "", "", status_msg - - status, video_name = fetch_video_full_dir(video_folder_full_dir, video_name) - - #if no video found, exit confirm_dirs and return status False - if not(status): - status_msg = "video file not found" - print(status_msg) - return False, "", "", "", "", status_msg - - # finally if no error found, return all full paths - status_msg = "Root and child dirs are all correct" - print(status_msg) - return True, know_faces_full_dir, unknown_faces_full_dir, video_folder_full_dir, video_name, status_msg - -# get video full path by name or get 1st video in the folder -def fetch_video_full_dir(video_dir, video_name): - video_dir_files = os.listdir(video_dir) - print("-----video for analysis", video_dir_files[0] if video_name==None else video_name) - if len(video_dir_files) > 0: - video_name = video_dir_files[0] if video_name==None else video_name - video_file_full_dir = os.path.normpath(os.path.join(video_dir, video_name)) - status = os.path.isfile(video_file_full_dir) - else: - video_file_full_dir = "empty" - status = False - return status, video_name - -# load the faces from the known_face_dir folder and get their names -def load_faces(known_face_dir, model, num_jitters): - known_encodings = [] - known_names = [] - for name in os.listdir(known_face_dir): - current_person_dir = os.path.normpath(os.path.join(known_face_dir, name)) - # Load every file of faces of known person - for filename in os.listdir(current_person_dir): - current_person_face_dir = os.path.normpath(os.path.join(current_person_dir, filename)) - # Load an image - image = face_recognition.load_image_file(current_person_face_dir) - - # Encoding, return list encoding for EACH face found in photo - encoding = face_recognition.face_encodings(image, num_jitters=num_jitters, model=model)[0] - - # Append encodings and name - known_encodings.append(encoding) - known_names.append(name) - return known_encodings, known_names - -# used to label the frames for faces and show the current frame on screen -def show_labeled_image(frame, show_video_output, n_faces, face_locations, resiz_factor, face_names): - for (top, right, bottom, left), name in zip(face_locations, face_names): - - top = int(top//resiz_factor) - right = int(right//resiz_factor) - bottom = int(bottom//resiz_factor) - left = int(left//resiz_factor) - # Draw a box around the face - cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2) - - # Draw a label with a name below the face - cv2.rectangle(frame, (left, bottom - 35), (right, bottom), (0, 0, 255), cv2.FILLED) - font = cv2.FONT_HERSHEY_DUPLEX - cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1) - cv2.putText(frame, f"{n_faces} Faces", (12, 40), font, 0.7, (255, 255, 255), 1) - - # Display the resulting image - - if show_video_output: - cv2.imshow('Video', frame) - - return frame - -# the inference -def video_inference(known_encodings, known_names, video_folder, video_name, unknown_faces_dir, model, skip_frames, n_upscale, resiz_factor,show_video_output, write_video_output, tolerance): - unknowns = 0 - knowns = 0 - video_file = os.path.normpath(os.path.join(video_folder, video_name)) - video_name = video_name.split('.')[0] - faces_in_frames = {} - faces_all_timestamps = {} - frame_array = [] - video_capture = cv2.VideoCapture(video_file) - totalNoFrames = video_capture.get(cv2.CAP_PROP_FRAME_COUNT) - fps = video_capture.get(cv2.CAP_PROP_FPS) - - - frame_counter = 1 - print("-------processing: ", video_file) - - while True: - # Grab a single frame of video - frame_exist, frame = video_capture.read() - timestamp = int(video_capture.get(cv2.CAP_PROP_POS_MSEC)) - if timestamp == 0: - timestamp = int(1000 * (totalNoFrames / fps)) - time_m_s = f"{timestamp}MSec : {timestamp//60000}m:{int((timestamp%60000)/1000)}s" - if not(frame_exist): - print("Video finished at ", time_m_s) - break - - # Only process every other frame of video to save time - if frame_counter%skip_frames==0: - height, width, layers = frame.shape - size = (width,height) - - - print(f"time: {time_m_s}", end ='') - # Resize frame of video to 1/4 size for faster face recognition processing - resized_frame = cv2.resize(frame, (0, 0), fx=resiz_factor, fy=resiz_factor) - - # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses) - # RGB_frame = resized_frame[:, :, ::-1] - RGB_frame = resized_frame - - # Find all the faces and face encodings in the current frame of video - face_locations = face_recognition.face_locations(RGB_frame, number_of_times_to_upsample=n_upscale, model=model) - face_encodings = face_recognition.face_encodings(RGB_frame, face_locations) - n_faces = len(face_locations) - print(f" - Found {n_faces} faces", end ='') - - face_names = [] - for i, (face_encoding, face_location) in enumerate(zip(face_encodings, face_locations)): - # See if the face is a match for the known face(s) - matches = face_recognition.compare_faces(known_encodings, face_encoding, tolerance=tolerance) - top, right, bottom, left = face_location - top = int(top//resiz_factor) - right = int(right//resiz_factor) - bottom = int(bottom//resiz_factor) - left = int(left//resiz_factor) - face_image = frame[top:bottom, left:right] - - # # If a match was found in known_face_encodings, just use the first one. - # if True in matches: - # first_match_index = matches.index(True) - # name = known_face_names[first_match_index] - - # Or instead, use the known face with the smallest distance to the new face - face_distances = face_recognition.face_distance(known_encodings, face_encoding) - best_match_index = np.argmin(face_distances) - if matches[best_match_index]: - name = known_names[best_match_index] - knowns+=1 - # create two dic, faces_in_frames: first time face appear in video, faces_all_timestamps: all times face appear in video - if name not in faces_in_frames.keys(): - faces_all_timestamps[name] = [timestamp] - faces_in_frames[name] = timestamp - else: - faces_all_timestamps[name].append(timestamp) - - else: - name = "Unknown" - unknowns+=1 - image_name = f"time{timestamp}_unknown.jpeg" - save_photo(unknown_faces_dir, video_name, face_image, image_name) - face_names.append(name) - print(f", recognized: {name}", end = '') - print() - - frame = show_labeled_image(frame, show_video_output, n_faces, face_locations, resiz_factor, face_names) - frame_array.append(frame) - - frame_counter +=1 - - - # Hit 'q' on the keyboard to quit! - if cv2.waitKey(1) & 0xFF == ord('q'): - break - - faces_split_timestamps = split_show_time(faces_all_timestamps) - - # fps for output file, if it's less than 1 the files is corrupted - if skip_frames > 0.5*fps: - target_fps=int(2*fps/skip_frames) - else: - target_fps=int(fps/skip_frames) - if target_fps < 2: # min fps = 2 - target_fps = 2 - # writes file - if write_video_output: - save_video(video_name, unknown_faces_dir, frame_array, target_fps=target_fps, image_size=size) - - # Release handle to the webcam - video_capture.release() - cv2.destroyAllWindows() - - total = unknowns + knowns - return faces_in_frames, total, unknowns, knowns, faces_split_timestamps - -# save photos in folder -def save_photo(unknown_faces_dir, video_name, face_image, image_name): - unkowndir_per_video_dir = os.path.normpath(os.path.join(unknown_faces_dir, video_name)) - if not(os.path.isdir(unkowndir_per_video_dir)): - os.mkdir(unkowndir_per_video_dir) - full_img_dir = os.path.normpath(os.path.join(unkowndir_per_video_dir, image_name)) - cv2.imwrite(full_img_dir ,face_image) - -# save video in folder -def save_video(video_name, video_folder, frame_array, target_fps, image_size): - out_path = os.path.normpath(os.path.join(video_folder, video_name)) - if not(os.path.isdir(out_path)): - os.mkdir(out_path) - out_name = f"{video_name}_labeled.avi" - out_path = os.path.normpath(os.path.join(out_path, out_name)) - print("----output video: ", out_path, "fps: ", {target_fps}) - out_writer = cv2.VideoWriter(out_path, cv2.VideoWriter_fourcc(*'XVID'), target_fps, image_size) - for i in range(len(frame_array)): - # writing to a image array - out_writer.write(frame_array[i]) - out_writer.release() - -# split continous milliseconds into seperate entries of start and end -def split_show_time(named_show, exit_threshold_ms = 1500): - named_split_shows = {} - for name, arr in named_show.items(): - temp = [arr[0]] - splits = [] - - for i, v in enumerate(arr): - if v == temp[-1]: - pass - elif (v - temp[-1] <= exit_threshold_ms): - temp.append(v) - elif not(i == len(arr)): - splits.append(temp) - temp = [v] - else: - splits.append(temp) - - # append last temp after for exit - splits.append(temp) - - # keep start and end only, append dummy value if an array has single frame - for i, times in enumerate(splits): - if len(times) == 1: - times.append(times[-1]) - splits[i] = [times[0], times[-1]] - - named_split_shows[name] = splits - - named_split_shows = refine_results(named_split_shows) - return named_split_shows - -# remove single frame faces before the split_show_time return results -def refine_results(named_split_shows): - for name, arrs in list(named_split_shows.items()): - if (len(arrs) == 1) and (arrs[0][0] == arrs[0][-1]): - named_split_shows.pop(name) - - return named_split_shows - -# activate the inference method and produce the results -def pipeline( - model = 'hog', # 'hog': faster, less acurate - or - 'cnn': slower, more accurate - skip_frames=5, # higher = faster, less acurate - n_upscale=1, # higher = slower, more accurate (min 1) - resiz_factor=1, # higher = slower, more accurate (min 0) - num_jitters=1, # higher = slower, more accurate (min 0) - tolerance=0.6, # lower = more accurate but less matching (min 0) - show_video_output=False, - write_video_output=True, - video_name = None, - ): - - status, know_faces_dir, unknown_faces_dir, video_dir, video_name, msg = confirm_dirs(DOCKER_DIR, video_name) - status_code = 0 - - if status: - try: - known_encodings, known_names = load_faces(know_faces_dir, model=model, num_jitters=num_jitters) - except Exception as e: - status_code = 427 # error in loading faces - status_msg = "error in loading faces" - print("-----error22: ",e) - obj = {"status": status_msg,} - return obj, status_code - - try: - faces_in_frames, total, unknowns, knowns ,faces_split_timestamps= video_inference( - known_encodings=known_encodings, - known_names=known_names, - video_folder=video_dir, - video_name=video_name, - unknown_faces_dir=unknown_faces_dir, - model = model, - skip_frames=skip_frames, - n_upscale=n_upscale, - resiz_factor=resiz_factor, - show_video_output=show_video_output, - write_video_output=write_video_output, - tolerance=tolerance, - ) - - faces_list = [] - for key, val in faces_in_frames.items(): - person = {"name": key, - "time": val} - faces_list.append(person) - - named_split_shows = [] - for key, value in faces_split_timestamps.items(): - entry = { - "name": key , - "appearances_count": len(value), - "appearances_details": [], - } - - for arr in value: - temp = {"entry_start": arr[0], "entry_finish": arr[-1]} - entry["appearances_details"].append(temp) - - named_split_shows.append(entry) - - status_msg = "done" - status_code = 200 - obj = {"status": status_msg, - "video_file": video_name, - "total_faces_count": total, - "unknown_faces_count": unknowns, - "known_faces_count": knowns, - "known_faces_list": faces_list, - "faces_split_timestamps": named_split_shows, - } - return obj, status_code - except Exception as e: - print("-----error33: ", e) - status_code = 428 # error in detecting faces - status_msg = "error in detecting faces" - obj = {"status": status_msg,} - return obj, status_code - else: - status_code = 426 # error in loading dirs - obj = {"status": msg,} - return obj, status_code - - -# pipeline(video_name="3.mp4", skip_frames=30, resiz_factor=1, model="hog") -# pipeline() +import face_recognition +import os +import cv2 +import numpy as np +import dlib +# import time + +APP_DIR = os.path.dirname(__file__) +DOCKER_DIR = os.path.dirname(APP_DIR) + +# confirm required dirs exist and append the current dir to the required folders name to produce absolute paths +def confirm_dirs(parent_dir, video_name): + mount_relative_dir = 'face_rec_files' + know_faces_relative_dir = 'known' + unknown_faces_relative_dir = 'unknown' + video_relative_dir = "feed" + + mount_full_dir = os.path.normpath(os.path.join(parent_dir, mount_relative_dir)) + mount_exist = os.path.isdir(mount_full_dir) + + status_msg = "" + + #if error in mount folder, exit confirm_dirs and return status False + if not(mount_exist): + status_msg = "root working directory not found" + print(status_msg) + return False, "", "", "", "", status_msg + + # load all directories and get full paths + try: + know_faces_full_dir = os.path.normpath(os.path.join(mount_full_dir, know_faces_relative_dir)) + unknown_faces_full_dir = os.path.normpath(os.path.join(mount_full_dir, unknown_faces_relative_dir)) + video_folder_full_dir = os.path.normpath(os.path.join(mount_full_dir, video_relative_dir)) + + know_faces_full_dir_exist = os.path.isdir(know_faces_full_dir) + unknown_faces_full_dir_exist = os.path.isdir(unknown_faces_full_dir) + video_folder_full_dir_exist = os.path.isdir(video_folder_full_dir) + all_dir_exist = know_faces_full_dir_exist and unknown_faces_full_dir_exist and video_folder_full_dir_exist + except Exception as e: + print("-----error11: ", e) + status_msg = "unexpected error, please check all directories and videos" + return False, "", "", "", "", status_msg + + #if error in directories structure, exit confirm_dirs and return status False + if not(all_dir_exist): + status_msg = f"Error in dirs - known: {know_faces_full_dir_exist}, unknown: {unknown_faces_full_dir_exist}, video: {video_folder_full_dir_exist}" + print(status_msg) + return False, "", "", "", "", status_msg + + status, video_name = fetch_video_full_dir(video_folder_full_dir, video_name) + + #if no video found, exit confirm_dirs and return status False + if not(status): + status_msg = "video file not found" + print(status_msg) + return False, "", "", "", "", status_msg + + # finally if no error found, return all full paths + status_msg = "Root and child dirs are all correct" + print(status_msg) + return True, know_faces_full_dir, unknown_faces_full_dir, video_folder_full_dir, video_name, status_msg + +# get video full path by name or get 1st video in the folder +def fetch_video_full_dir(video_dir, video_name): + video_dir_files = os.listdir(video_dir) + print("-----video for analysis", video_dir_files[0] if video_name==None else video_name) + if len(video_dir_files) > 0: + video_name = video_dir_files[0] if video_name==None else video_name + video_file_full_dir = os.path.normpath(os.path.join(video_dir, video_name)) + status = os.path.isfile(video_file_full_dir) + else: + video_file_full_dir = "empty" + status = False + return status, video_name + +# load the faces from the known_face_dir folder and get their names +def load_faces(known_face_dir, model, num_jitters): + known_encodings = [] + known_names = [] + for name in os.listdir(known_face_dir): + current_person_dir = os.path.normpath(os.path.join(known_face_dir, name)) + # Load every file of faces of known person + for filename in os.listdir(current_person_dir): + current_person_face_dir = os.path.normpath(os.path.join(current_person_dir, filename)) + # Load an image + image = face_recognition.load_image_file(current_person_face_dir) + + # Encoding, return list encoding for EACH face found in photo + encoding = face_recognition.face_encodings(image, num_jitters=num_jitters, model=model)[0] + + # Append encodings and name + known_encodings.append(encoding) + known_names.append(name) + return known_encodings, known_names + +# used to label the frames for faces and show the current frame on screen +def show_labeled_image(frame, show_video_output, n_faces, face_locations, resiz_factor, face_names): + for (top, right, bottom, left), name in zip(face_locations, face_names): + + top = int(top//resiz_factor) + right = int(right//resiz_factor) + bottom = int(bottom//resiz_factor) + left = int(left//resiz_factor) + # Draw a box around the face + cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2) + + # Draw a label with a name below the face + cv2.rectangle(frame, (left, bottom - 35), (right, bottom), (0, 0, 255), cv2.FILLED) + font = cv2.FONT_HERSHEY_DUPLEX + cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1) + cv2.putText(frame, f"{n_faces} Faces", (12, 40), font, 0.7, (255, 255, 255), 1) + + # Display the resulting image + + if show_video_output: + cv2.imshow('Video', frame) + + return frame + +# the inference +def video_inference(known_encodings, known_names, video_folder, video_name, unknown_faces_dir, model, skip_frames, n_upscale, resiz_factor,show_video_output, write_video_output, tolerance): + unknowns = 0 + knowns = 0 + video_file = os.path.normpath(os.path.join(video_folder, video_name)) + video_name = video_name.split('.')[0] + faces_in_frames = {} + faces_all_timestamps = {} + frame_array = [] + video_capture = cv2.VideoCapture(video_file) + totalNoFrames = video_capture.get(cv2.CAP_PROP_FRAME_COUNT) + fps = video_capture.get(cv2.CAP_PROP_FPS) + + + frame_counter = 1 + print("-------processing: ", video_file) + + while True: + # Grab a single frame of video + frame_exist, frame = video_capture.read() + timestamp = int(video_capture.get(cv2.CAP_PROP_POS_MSEC)) + if timestamp == 0: + timestamp = int(1000 * (totalNoFrames / fps)) + time_m_s = f"{timestamp}MSec : {timestamp//60000}m:{int((timestamp%60000)/1000)}s" + if not(frame_exist): + print("Video finished at ", time_m_s) + break + + # Only process every other frame of video to save time + if frame_counter%skip_frames==0: + height, width, layers = frame.shape + size = (width,height) + + + print(f"time: {time_m_s}", end ='') + # Resize frame of video to 1/4 size for faster face recognition processing + resized_frame = cv2.resize(frame, (0, 0), fx=resiz_factor, fy=resiz_factor) + + # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses) + # RGB_frame = resized_frame[:, :, ::-1] + RGB_frame = resized_frame + + # Find all the faces and face encodings in the current frame of video + face_locations = face_recognition.face_locations(RGB_frame, number_of_times_to_upsample=n_upscale, model=model) + face_encodings = face_recognition.face_encodings(RGB_frame, face_locations) + n_faces = len(face_locations) + print(f" - Found {n_faces} faces", end ='') + + face_names = [] + for i, (face_encoding, face_location) in enumerate(zip(face_encodings, face_locations)): + # See if the face is a match for the known face(s) + matches = face_recognition.compare_faces(known_encodings, face_encoding, tolerance=tolerance) + top, right, bottom, left = face_location + top = int(top//resiz_factor) + right = int(right//resiz_factor) + bottom = int(bottom//resiz_factor) + left = int(left//resiz_factor) + face_image = frame[top:bottom, left:right] + + # # If a match was found in known_face_encodings, just use the first one. + # if True in matches: + # first_match_index = matches.index(True) + # name = known_face_names[first_match_index] + + # Or instead, use the known face with the smallest distance to the new face + face_distances = face_recognition.face_distance(known_encodings, face_encoding) + best_match_index = np.argmin(face_distances) + if matches[best_match_index]: + name = known_names[best_match_index] + knowns+=1 + # create two dic, faces_in_frames: first time face appear in video, faces_all_timestamps: all times face appear in video + if name not in faces_in_frames.keys(): + faces_all_timestamps[name] = [timestamp] + faces_in_frames[name] = timestamp + else: + faces_all_timestamps[name].append(timestamp) + + else: + name = "Unknown" + unknowns+=1 + image_name = f"time{timestamp}_unknown.jpeg" + save_photo(unknown_faces_dir, video_name, face_image, image_name) + face_names.append(name) + print(f", recognized: {name}", end = '') + print() + + frame = show_labeled_image(frame, show_video_output, n_faces, face_locations, resiz_factor, face_names) + frame_array.append(frame) + + frame_counter +=1 + + + # Hit 'q' on the keyboard to quit! + if cv2.waitKey(1) & 0xFF == ord('q'): + break + + faces_split_timestamps = split_show_time(faces_all_timestamps) + + # fps for output file, if it's less than 1 the files is corrupted + if skip_frames > 0.5*fps: + target_fps=int(2*fps/skip_frames) + else: + target_fps=int(fps/skip_frames) + if target_fps < 2: # min fps = 2 + target_fps = 2 + # writes file + if write_video_output: + save_video(video_name, unknown_faces_dir, frame_array, target_fps=target_fps, image_size=size) + + # Release handle to the webcam + video_capture.release() + cv2.destroyAllWindows() + + total = unknowns + knowns + return faces_in_frames, total, unknowns, knowns, faces_split_timestamps + +# save photos in folder +def save_photo(unknown_faces_dir, video_name, face_image, image_name): + unkowndir_per_video_dir = os.path.normpath(os.path.join(unknown_faces_dir, video_name)) + if not(os.path.isdir(unkowndir_per_video_dir)): + os.mkdir(unkowndir_per_video_dir) + full_img_dir = os.path.normpath(os.path.join(unkowndir_per_video_dir, image_name)) + cv2.imwrite(full_img_dir ,face_image) + +# save video in folder +def save_video(video_name, video_folder, frame_array, target_fps, image_size): + out_path = os.path.normpath(os.path.join(video_folder, video_name)) + if not(os.path.isdir(out_path)): + os.mkdir(out_path) + out_name = f"{video_name}_labeled.avi" + out_path = os.path.normpath(os.path.join(out_path, out_name)) + print("----output video: ", out_path, "fps: ", {target_fps}) + out_writer = cv2.VideoWriter(out_path, cv2.VideoWriter_fourcc(*'XVID'), target_fps, image_size) + for i in range(len(frame_array)): + # writing to a image array + out_writer.write(frame_array[i]) + out_writer.release() + +# split continous milliseconds into seperate entries of start and end +def split_show_time(named_show, exit_threshold_ms = 1500): + named_split_shows = {} + for name, arr in named_show.items(): + temp = [arr[0]] + splits = [] + + for i, v in enumerate(arr): + if v == temp[-1]: + pass + elif (v - temp[-1] <= exit_threshold_ms): + temp.append(v) + elif not(i == len(arr)): + splits.append(temp) + temp = [v] + else: + splits.append(temp) + + # append last temp after for exit + splits.append(temp) + + # keep start and end only, append dummy value if an array has single frame + for i, times in enumerate(splits): + if len(times) == 1: + times.append(times[-1]) + splits[i] = [times[0], times[-1]] + + named_split_shows[name] = splits + + named_split_shows = refine_results(named_split_shows) + return named_split_shows + +# remove single frame faces before the split_show_time return results +def refine_results(named_split_shows): + for name, arrs in list(named_split_shows.items()): + if (len(arrs) == 1) and (arrs[0][0] == arrs[0][-1]): + named_split_shows.pop(name) + + return named_split_shows + +# activate the inference method and produce the results +def pipeline( + model = 'hog', # 'hog': faster, less acurate - or - 'cnn': slower, more accurate + skip_frames=5, # higher = faster, less acurate + n_upscale=1, # higher = slower, more accurate (min 1) + resiz_factor=1, # higher = slower, more accurate (min 0) + num_jitters=1, # higher = slower, more accurate (min 0) + tolerance=0.6, # lower = more accurate but less matching (min 0) + show_video_output=False, + write_video_output=True, + video_name = None, + ): + + status, know_faces_dir, unknown_faces_dir, video_dir, video_name, msg = confirm_dirs(DOCKER_DIR, video_name) + status_code = 0 + + if status: + try: + known_encodings, known_names = load_faces(know_faces_dir, model=model, num_jitters=num_jitters) + except Exception as e: + status_code = 427 # error in loading faces + status_msg = "error in loading faces" + print("-----error22: ",e) + obj = {"status": status_msg,} + return obj, status_code + + try: + faces_in_frames, total, unknowns, knowns ,faces_split_timestamps= video_inference( + known_encodings=known_encodings, + known_names=known_names, + video_folder=video_dir, + video_name=video_name, + unknown_faces_dir=unknown_faces_dir, + model = model, + skip_frames=skip_frames, + n_upscale=n_upscale, + resiz_factor=resiz_factor, + show_video_output=show_video_output, + write_video_output=write_video_output, + tolerance=tolerance, + ) + + faces_list = [] + for key, val in faces_in_frames.items(): + person = {"name": key, + "time": val} + faces_list.append(person) + + named_split_shows = [] + for key, value in faces_split_timestamps.items(): + entry = { + "name": key , + "appearances_count": len(value), + "appearances_details": [], + } + + for arr in value: + temp = {"entry_start": arr[0], "entry_finish": arr[-1]} + entry["appearances_details"].append(temp) + + named_split_shows.append(entry) + + status_msg = "done" + status_code = 200 + obj = {"status": status_msg, + "video_file": video_name, + "total_faces_count": total, + "unknown_faces_count": unknowns, + "known_faces_count": knowns, + "known_faces_list": faces_list, + "faces_split_timestamps": named_split_shows, + } + return obj, status_code + except Exception as e: + print("-----error33: ", e) + status_code = 428 # error in detecting faces + status_msg = "error in detecting faces" + obj = {"status": status_msg,} + return obj, status_code + else: + status_code = 426 # error in loading dirs + obj = {"status": msg,} + return obj, status_code + + +# pipeline(video_name="3.mp4", skip_frames=30, resiz_factor=1, model="hog") +# pipeline() diff --git a/readme.md b/readme.md index f100531..b12b15c 100644 --- a/readme.md +++ b/readme.md @@ -1,129 +1,129 @@ -# Face Recognition -Face recognition model: detects multiple faces in a video and recognize known persons. -faces pictures, names, video has to follow certain directory structure: -``` -|-- face_rec_files: root folder mounted to the docker -| |-- feed: contain the videos for the analysis -| |-- unknown: the result of the unidentified faces will be stored here -| |-- known: contain folders for people to recognize -| | |-- person_1: Name of person_1, all person_1 photos are inside the folder -| | |-- person_2: Name of person_2, all person_2 photos are inside the folder -| | |-- person_x: Name of person_x, all person_x photos are inside the folder -``` - -## Building the docker -`docker build -f ./Dockerfile -t face_rec .` - -## Running docker -run the docker with a mount option pointing to a local folder on the host machine, this has to follow the structure mentioned above - -`docker run -it -p 3000:3000 --rm --name face_rec --mount type=bind,source=C:/path/to/mounted_folder,target=/app/face_rec_files face_rec` -``` --it: interactive, --p: exposed port, ---rm: remove after stop, ---name: name of the container, ---mount: mount external location from host to the container -``` -## Request service -3 request are available in the docker -- `healthcheck` - GET -- `run_defaults` - GET -- `run_custom` - POST - -request can be used with url: `http://hostip:3000/method_name` - -## GET /healthchech - -checks if the docker is running and listning to the correct port, and check if the mounted folder exist - -**GET /healthchech response** -``` -json = { - "status": "current status" - } -& status_code -``` - -status_code & status values: -- 200: ready -- 402: mount folder not found - -## GET /run_defaults -activates the analysis by default parameters and analyze the first video in the feed folder then return response object and the output files - -## POST /run_custom -activates the analysis by custom parameters and custom video file from the feed folder then return response object and the output files - -**POST /run_custom request object** -A json object containing all the required parameters has to be passed through the POST request. -sample request: `requests.post(host, json=parameters_json)` - -parameters_json keys and values: -not all keys has to be passed, the server will replace the missing keys from the request with default values, it's recommended to leave the default values except for the file name and skip frames, the rest is just used to fine tune the matching or more advanced usage. -``` -parameters_json = { - "file_name": "2.mp4", # video file to be used for the analysis from the feed folder - "skip_frames": 10, # frames to skip while analyzing, higher skip = faster analysis. possible values: [0, 10, 20, ...] - "resiz_factor": 1, # resize factor for the video image size, lower resiz_factor = faster analysis but lower accurecy. possible values: [0.1, 0.5, 1.5, ...] - "num_jitters": 1, how many times to sample the known pictures, higher jitter = slower analysis but higher accurecy. possible values: [0, 1, 2, ...] - "tolerance": 0.6, tolerance for matching faces with known people, higher tolerance = more matches but lower accurecy. possible values: [0, 0.4, 0.8, 1] - "model": "hog", # model to be used, either "hog" for CPU analysis or "cnn" for GPU, GPU might not be accessible through the docker . possible values: ["hog", "cnn] - } -``` - - -## /run response -response is in json format accessed from response[text], -`status`: did the analysis complete successfully -`video_file`: full directory of file used for the analysis -`known_faces_count`: number of faces detected and recognized (count duplicates) -`unknown_faces_count`: number of faces detected but not recognized (count duplicates) -`total_faces_count`: number of all faces detected (count duplicates) -`known_faces_list`: list of recognized faces and their first appearance in MSec -`faces_split_timestamps`: list of recognized faces and every appearance start and end in MSec - faces recognized in a single frame are removed from this list (most likely false detections) -``` -Sample return json = { - "status":"done", - "video_file": full/path/video_file.mp4, - "known_faces_count": 26, - "unknown_faces_count":18 - "total_faces_count":44, - "known_faces_list": - [ - {"name":"thor","time":7465}, - {"name":"loki","time":10385} - ], - "faces_split_timestamps": [ - { - "name": "thor", - "appearances_count": 2, - "appearances_details": [ - { - "entry_start": 7160, - "entry_finish": 7160 - }, - { - "entry_start": 14360, - "entry_finish": 14360 - } - ] - }, - { - "name": "loki", - "appearances_count": 1, - "appearances_details": [ - { - "entry_start": 7160, - "entry_finish": 7160 - } - ] - }, - ] - } -& status_code -``` - -**/run output files** -files created by the model after it finishes processing: +# Face Recognition +Face recognition model: detects multiple faces in a video and recognize known persons. +faces pictures, names, video has to follow certain directory structure: +``` +|-- face_rec_files: root folder mounted to the docker +| |-- feed: contain the videos for the analysis +| |-- unknown: the result of the unidentified faces will be stored here +| |-- known: contain folders for people to recognize +| | |-- person_1: Name of person_1, all person_1 photos are inside the folder +| | |-- person_2: Name of person_2, all person_2 photos are inside the folder +| | |-- person_x: Name of person_x, all person_x photos are inside the folder +``` + +## Building the docker +`docker build -f ./Dockerfile -t face_rec .` + +## Running docker +run the docker with a mount option pointing to a local folder on the host machine, this has to follow the structure mentioned above + +`docker run -it -p 3000:3000 --rm --name face_rec --mount type=bind,source=C:/path/to/mounted_folder,target=/app/face_rec_files face_rec` +``` +-it: interactive, +-p: exposed port, +--rm: remove after stop, +--name: name of the container, +--mount: mount external location from host to the container +``` +## Request service +3 request are available in the docker +- `healthcheck` - GET +- `run_defaults` - GET +- `run_custom` - POST + +request can be used with url: `http://hostip:3000/method_name` + +## GET /healthchech + +checks if the docker is running and listning to the correct port, and check if the mounted folder exist + +**GET /healthchech response** +``` +json = { + "status": "current status" + } +& status_code +``` + +status_code & status values: +- 200: ready +- 402: mount folder not found + +## GET /run_defaults +activates the analysis by default parameters and analyze the first video in the feed folder then return response object and the output files + +## POST /run_custom +activates the analysis by custom parameters and custom video file from the feed folder then return response object and the output files + +**POST /run_custom request object** +A json object containing all the required parameters has to be passed through the POST request. +sample request: `requests.post(host, json=parameters_json)` + +parameters_json keys and values: +not all keys has to be passed, the server will replace the missing keys from the request with default values, it's recommended to leave the default values except for the file name and skip frames, the rest is just used to fine tune the matching or more advanced usage. +``` +parameters_json = { + "file_name": "2.mp4", # video file to be used for the analysis from the feed folder + "skip_frames": 10, # frames to skip while analyzing, higher skip = faster analysis. possible values: [0, 10, 20, ...] + "resiz_factor": 1, # resize factor for the video image size, lower resiz_factor = faster analysis but lower accurecy. possible values: [0.1, 0.5, 1.5, ...] + "num_jitters": 1, how many times to sample the known pictures, higher jitter = slower analysis but higher accurecy. possible values: [0, 1, 2, ...] + "tolerance": 0.6, tolerance for matching faces with known people, higher tolerance = more matches but lower accurecy. possible values: [0, 0.4, 0.8, 1] + "model": "hog", # model to be used, either "hog" for CPU analysis or "cnn" for GPU, GPU might not be accessible through the docker . possible values: ["hog", "cnn] + } +``` + + +## /run response +response is in json format accessed from response[text], +`status`: did the analysis complete successfully +`video_file`: full directory of file used for the analysis +`known_faces_count`: number of faces detected and recognized (count duplicates) +`unknown_faces_count`: number of faces detected but not recognized (count duplicates) +`total_faces_count`: number of all faces detected (count duplicates) +`known_faces_list`: list of recognized faces and their first appearance in MSec +`faces_split_timestamps`: list of recognized faces and every appearance start and end in MSec - faces recognized in a single frame are removed from this list (most likely false detections) +``` +Sample return json = { + "status":"done", + "video_file": full/path/video_file.mp4, + "known_faces_count": 26, + "unknown_faces_count":18 + "total_faces_count":44, + "known_faces_list": + [ + {"name":"thor","time":7465}, + {"name":"loki","time":10385} + ], + "faces_split_timestamps": [ + { + "name": "thor", + "appearances_count": 2, + "appearances_details": [ + { + "entry_start": 7160, + "entry_finish": 7160 + }, + { + "entry_start": 14360, + "entry_finish": 14360 + } + ] + }, + { + "name": "loki", + "appearances_count": 1, + "appearances_details": [ + { + "entry_start": 7160, + "entry_finish": 7160 + } + ] + }, + ] + } +& status_code +``` + +**/run output files** +files created by the model after it finishes processing: - in unknown folder: creates a new folder by the name of the video containing images for each detected but unrecognized face and a labeled video \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 595b0d2..d47f26e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -Flask==2.1.3 -numpy==1.23.1 -pandas==1.4.3 -dlib==19.24.0 -gevent==21.8.0 +Flask==2.1.3 +numpy==1.23.1 +pandas==1.4.3 +dlib==19.24.0 +gevent==21.8.0 face_recognition==1.3.0 \ No newline at end of file