added return faces_split_timestamps

changelog.md

@@ -1,5 +1,12 @@
 # Changelog
-## 24-Sep-22
+
+## 25-Sep-22 - git:feature-start_end
+1. added `video_face_rec.py/faces_split_timestamps` return value for all the enteries a faces appears
+2. added `video_face_rec.py/refine_results` to remove single frames faces from enteries return, but not activated in 1st appearance return
+3. fixed `timestamp` = 0 by the end of the video
+4. fixed `target_fps` having value less than 1 which resulted in corrupted video output
+
+## 24-Sep-22 - git:feature-custm_analysis
 1. restructured the files into folders
     - `/face_rec_code/`: for all the python code
     - `/face_rec_files/`: for the videos and pictures

face_rec_code/app.py

@@ -8,6 +8,7 @@
 warnings.simplefilter("ignore")
 
 app = Flask(__name__)
+app.config['JSON_SORT_KEYS'] = False
 APP_DIR = os.path.dirname(__file__)
 DOCKER_DIR = os.path.dirname(APP_DIR)
 

face_rec_code/request_test.py

@@ -5,7 +5,7 @@
 post_obj = {
             "file_name": "3.mp4",
             # "model": "hog",
-            "skip_frames": 60,
+            "skip_frames": 30,
             # "resiz_factor": 1,
             # "num_jitters": 1,
             # "tolerance": 0.6,

face_rec_code/video_face_rec.py

@@ -3,7 +3,7 @@
 import cv2
 import numpy as np
 import dlib
-import time
+# import time
 
 APP_DIR = os.path.dirname(__file__)
 DOCKER_DIR = os.path.dirname(APP_DIR)
@@ -124,27 +124,33 @@ def video_inference(known_encodings, known_names, video_folder, video_name, unkn
     video_file = os.path.normpath(os.path.join(video_folder, video_name))
     video_name = video_name.split('.')[0]
     faces_in_frames = {}
+    faces_all_timestamps = {}
     frame_array = []
     video_capture = cv2.VideoCapture(video_file)
+    totalNoFrames = video_capture.get(cv2.CAP_PROP_FRAME_COUNT)
+    fps = video_capture.get(cv2.CAP_PROP_FPS)
+
+
     frame_counter = 1
     print("-------processing: ", video_file)
 
     while True:
         # Grab a single frame of video
-        ret, frame = video_capture.read()
-        if not(ret):
-            print("Video finished at frame ", frame_counter)
+        frame_exist, frame = video_capture.read()
+        timestamp = int(video_capture.get(cv2.CAP_PROP_POS_MSEC))
+        if timestamp == 0:
+            timestamp = int(1000 * (totalNoFrames / fps))
+        time_m_s = f"{timestamp}MSec : {timestamp//60000}m:{int((timestamp%60000)/1000)}s"
+        if not(frame_exist):
+            print("Video finished at ", time_m_s)
             break
 
         # Only process every other frame of video to save time
         if frame_counter%skip_frames==0:
             height, width, layers = frame.shape
             size = (width,height)
-            timestamp = int(video_capture.get(cv2.CAP_PROP_POS_MSEC))
 
-            time_m_s = f"{timestamp//60000}m:{int((timestamp%60000)/1000)}s"
-
-            fps = video_capture.get(cv2.CAP_PROP_FPS)
+
             print(f"time:  {time_m_s}", end ='')
             # Resize frame of video to 1/4 size for faster face recognition processing
             resized_frame = cv2.resize(frame, (0, 0), fx=resiz_factor, fy=resiz_factor)
@@ -181,8 +187,12 @@ def video_inference(known_encodings, known_names, video_folder, video_name, unkn
                 if matches[best_match_index]:
                     name = known_names[best_match_index]
                     knowns+=1
+                    # create two dic, faces_in_frames: first time face appear in video, faces_all_timestamps: all times face appear in video
                     if name not in faces_in_frames.keys():
-                        faces_in_frames[name]= timestamp
+                        faces_all_timestamps[name] = [timestamp]
+                        faces_in_frames[name] = timestamp
+                    else:
+                        faces_all_timestamps[name].append(timestamp)
 
                 else:
                     name = "Unknown"
@@ -194,24 +204,34 @@ def video_inference(known_encodings, known_names, video_folder, video_name, unkn
             print()
 
             frame = show_labeled_image(frame, show_video_output, n_faces, face_locations, resiz_factor, face_names)
-
             frame_array.append(frame)
+
         frame_counter +=1
 
 
         # Hit 'q' on the keyboard to quit!
         if cv2.waitKey(1) & 0xFF == ord('q'):
             break
 
+    faces_split_timestamps = split_show_time(faces_all_timestamps)
+
+    # fps for output file, if it's less than 1 the files is corrupted
+    if skip_frames > 0.5*fps:
+        target_fps=int(2*fps/skip_frames)
+    else:
+        target_fps=int(fps/skip_frames)
+    if target_fps < 2: # min fps = 2
+        target_fps = 2
+    # writes file
     if write_video_output:
-        save_video(video_name, unknown_faces_dir, frame_array, target_fps=int(fps/skip_frames), image_size=size)
+        save_video(video_name, unknown_faces_dir, frame_array, target_fps=target_fps, image_size=size)
 
     # Release handle to the webcam
     video_capture.release()
     cv2.destroyAllWindows()    
 
     total = unknowns + knowns
-    return faces_in_frames, total, unknowns, knowns
+    return faces_in_frames, total, unknowns, knowns, faces_split_timestamps
 
 # save photos in folder
 def save_photo(unknown_faces_dir, video_name, face_image, image_name):
@@ -228,13 +248,53 @@ def save_video(video_name, video_folder, frame_array, target_fps, image_size):
         os.mkdir(out_path)
     out_name = f"{video_name}_labeled.avi"
     out_path = os.path.normpath(os.path.join(out_path, out_name))
-    print("----output video: ", out_path)
+    print("----output video: ", out_path, "fps: ", {target_fps})
     out_writer = cv2.VideoWriter(out_path, cv2.VideoWriter_fourcc(*'XVID'), target_fps, image_size)
     for i in range(len(frame_array)):
         # writing to a image array
         out_writer.write(frame_array[i])
     out_writer.release()
 
+# split continous milliseconds into seperate entries of start and end
+def split_show_time(named_show, exit_threshold_ms = 1500):
+    named_split_shows = {}
+    for name, arr in named_show.items():
+        temp = [arr[0]]
+        splits = []
+
+        for i, v in enumerate(arr):
+            if v == temp[-1]:  
+                pass
+            elif (v - temp[-1] <= exit_threshold_ms):
+                temp.append(v)
+            elif not(i == len(arr)):
+                splits.append(temp)
+                temp = [v]
+            else:
+                splits.append(temp)
+
+        # append last temp after for exit
+        splits.append(temp)
+
+        # keep start and end only, append dummy value if an array has single frame
+        for i, times in enumerate(splits):
+            if len(times) == 1:
+                times.append(times[-1])
+            splits[i] = [times[0], times[-1]]
+
+        named_split_shows[name] = splits
+
+    named_split_shows = refine_results(named_split_shows)
+    return named_split_shows
+
+# remove single frame faces before the split_show_time return results
+def refine_results(named_split_shows):
+    for name, arrs in list(named_split_shows.items()):
+        if (len(arrs) == 1) and (arrs[0][0] == arrs[0][-1]):
+            named_split_shows.pop(name)
+
+    return named_split_shows
+
 # activate the inference method and produce the results
 def pipeline(
             model = 'hog', # 'hog': faster, less acurate - or - 'cnn': slower, more accurate
@@ -262,7 +322,7 @@ def pipeline(
             return obj, status_code
 
         try:  
-            faces_in_frames, total, unknowns, knowns = video_inference(
+            faces_in_frames, total, unknowns, knowns ,faces_split_timestamps= video_inference(
                 known_encodings=known_encodings, 
                 known_names=known_names, 
                 video_folder=video_dir, 
@@ -283,14 +343,29 @@ def pipeline(
                         "time": val}
                 faces_list.append(person)
 
+            named_split_shows = []
+            for key, value in faces_split_timestamps.items():
+                entry = {
+                        "name": key ,
+                        "appearances_count": len(value),
+                        "appearances_details": [],
+                        }
+
+                for arr in value:
+                    temp = {"entry_start": arr[0], "entry_finish": arr[-1]}
+                    entry["appearances_details"].append(temp)
+
+                named_split_shows.append(entry)
+
             status_msg = "done"
             status_code = 200
             obj = {"status": status_msg,
                     "video_file": video_name,
                     "total_faces_count": total,
                     "unknown_faces_count": unknowns,
                     "known_faces_count": knowns,
-                    "known_faces_list": faces_list
+                    "known_faces_list": faces_list,
+                    "faces_split_timestamps": named_split_shows,
                     }
             return obj, status_code
         except Exception as e:
@@ -305,5 +380,5 @@ def pipeline(
         return obj, status_code
 
 
-# pipeline(video_name="2.mp4", skip_frames=10)  
+# pipeline(video_name="3.mp4", skip_frames=30, resiz_factor=1, model="hog")  
 # pipeline()

readme.md

@@ -74,8 +74,16 @@ parameters_json = {
 
 
 ## /run response
+response is in json format accessed from response[text],  
+`status`: did the analysis complete successfully  
+`video_file`: full directory of file used for the analysis  
+`known_faces_count`: number of faces detected and recognized (count duplicates)  
+`unknown_faces_count`: number of faces detected but not recognized (count duplicates)  
+`total_faces_count`: number of all faces detected (count duplicates)  
+`known_faces_list`: list of recognized faces and their first appearance in MSec  
+`faces_split_timestamps`: list of recognized faces and every appearance start and end in MSec - faces recognized in a single frame are removed from this list (most likely false detections)  
 ```
-json = {
+Sample return json = {
         "status":"done",
         "video_file": full/path/video_file.mp4,
         "known_faces_count": 26,
@@ -86,6 +94,32 @@ json = {
                 {"name":"thor","time":7465},
                 {"name":"loki","time":10385}
             ],
+        "faces_split_timestamps": [
+        {
+            "name": "thor",
+            "appearances_count": 2,
+            "appearances_details": [
+                {
+                    "entry_start": 7160,
+                    "entry_finish": 7160
+                },
+                {
+                    "entry_start": 14360,
+                    "entry_finish": 14360
+                }
+            ]
+        },
+        {
+            "name": "loki",
+            "appearances_count": 1,
+            "appearances_details": [
+                {
+                    "entry_start": 7160,
+                    "entry_finish": 7160
+                }
+            ]
+        },
+    ]   
     }
 & status_code
 ```