Merge pull request #21 from ECLAIR-Robotics/number_detection

Changes so far to number detection

.gitignore

@@ -4,10 +4,7 @@ computer_vision/venv
 **/__pycache__/
 .DS_Store
 **/.DS_Store
-computer_vision/number_detection/__pycache__
-.DS_Store
-computer_vision/.DS_Store
-computer_vision/number_detection/.DS_Store
-computer_vision/number_detection/test/.DS_Store
-computer_vision/number_detection/test/__pycache__/
 computer_vision/number_detection/test/testResults/
+computer_vision/number_detection/test/test-cropped/
+computer_vision/number_detection/library/*
+!computer_vision/number_detection/library/

computer_vision/Makefile

@@ -75,7 +75,7 @@ endif
 
 CLEANUP_CMD := sudo rm $(MINICONDA_INSTALLER)
 
-.PHONY: install
+.PHONY: install 
 
 install:
 	@echo "Checking for package manager..."
@@ -106,4 +106,31 @@ install:
 	@eval "$$(conda shell.bash hook)"; \
 	conda activate pcr
 	@echo "Installing modules from 'requirements.txt'"
-	pip install -r "requirements.txt"
+
+	pip install -r "requirements.txt"
+
+SRC_FILE := ./number_detection/prefix_min.c
+LIB_DIR := ./number_detection/library
+CC := gcc
+
+ifeq ($(OS),Windows_NT)
+	LIB_FILE := $(LIB_DIR)/number-detection-pkg.dll
+	CFLAGS := -dynamiclib -o $(LIB_FILE)
+else 
+	ifeq ($(UNAME_S),Darwin)
+		LIB_FILE := $(LIB_DIR)/number-detection-pkg.dylib
+		CFLAGS := -dynamiclib -o $(LIB_FILE)
+	else ifeq ($(UNAME_S),Linux)
+		LIB_FILE := $(LIB_DIR)/number-detection-pkg.so
+		CFLAGS := -shared -o $(LIB_FILE)
+	else 
+		$(error Unsupported operating system: $(UNAME_S))
+	endif
+endif
+
+.PHONY: compile_lib
+compile_lib:
+	@echo "Compiling $(SRC_FILE) into $(LIB_FILE)..."
+	@mkdir -p $(LIB_DIR)
+	@$(CC) $(SRC_FILE) $(CFLAGS)
+	@echo "Compilation finished."

computer_vision/number_detection/crop_tool.py

@@ -0,0 +1,178 @@
+import cv2
+import numpy as np
+import threading
+import ctypes
+
+_initialized = False
+
+def init():
+    global lib
+    # Define the argument and return types of the C function
+    try:
+        lib = ctypes.CDLL('./library/number-detection-pkg.dylib')
+        lib.prefix_min.argtypes = [ctypes.POINTER(ctypes.c_uint8), ctypes.POINTER(ctypes.c_uint8), ctypes.c_int, ctypes.c_int, ctypes.c_int]
+        lib.prefix_min.restype = None
+        _initialized = True
+    except OSError as e:
+        print(f"Failed to load the shared library: {e}")
+        lib = None
+
+if not _initialized:
+    init()
+    _initialized = True
+
+
+def prefix_min(arr, results, axis=0):
+    if lib is None:
+        raise RuntimeError("Shared library not loaded. Cannot call prefix_min.")
+
+    arr_ctypes = arr.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
+    prefix_min_arr = np.empty_like(arr,dtype=np.uint8)
+    prefix_min_ctypes = prefix_min_arr.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
+
+    #call the C function
+
+    lib.prefix_min(arr_ctypes,prefix_min_ctypes,arr.shape[0],arr.shape[1],axis)
+
+    results[axis] = prefix_min_arr
+
+def get_mask(results:dict):
+    prefix_min_tb = results[0]
+    prefix_min_lr = results[1]
+    prefix_min_bt = results[2]
+    prefix_min_rl = results[3]
+
+    lowerThresh = 0
+    upperThresh = 25
+
+    combined_mask = ((prefix_min_tb >= lowerThresh) & (prefix_min_tb <= upperThresh)) & \
+            (prefix_min_lr >= lowerThresh) & (prefix_min_lr <= upperThresh) & \
+            (prefix_min_rl >= lowerThresh) & (prefix_min_rl <= upperThresh) & \
+            (prefix_min_bt >= lowerThresh) & (prefix_min_bt <= upperThresh) 
+
+    kernel = np.ones((5, 5), np.uint8)
+    combined_mask = cv2.dilate(combined_mask.astype(np.uint8), kernel, iterations=2)
+    combined_mask = cv2.erode(combined_mask, kernel, iterations=2)
+
+    return combined_mask
+
+
+#NOTE: this was code take from https://github.com/wjbmattingly/ocr_python_textbook/blob/main/02_02_working%20with%20opencv.ipynb
+
+def get_skew_angle(contour):
+
+    min_area_rect = cv2.minAreaRect(contour)
+    angle = min_area_rect[-1]
+    (h,w) = min_area_rect[1]
+
+    # Adjust the angle based on the w and height
+    if w < h:
+        tmp = h
+        h = w
+        w = tmp
+
+
+    if w > h:
+        angle = 90 - angle
+    if angle < -45:
+        angle = 90 + angle
+    elif angle > 45:
+        angle = angle - 90
+
+    # Round the angle to the nearest hundredth degree
+    angle = round(angle, 2)
+
+    return angle
+
+# Rotate the image around its center
+def rotate_image(cvImage, angle: float):
+    newImage = cvImage.copy()
+    (h, w) = newImage.shape[:2]
+    center = (w // 2, h // 2)
+    M = cv2.getRotationMatrix2D(center, angle, 1.0)
+    newImage = cv2.warpAffine(newImage, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)
+    return newImage
+
+def crop_image(img):
+    # Call the function and display the result
+
+    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+    results = {}
+    #NOTE: changed to 4 to calculate all scan direction 
+    NUM_THREADS = 4
+    threads = [None] * NUM_THREADS
+
+    # create and start threads
+    for i in range(0,NUM_THREADS):
+        threads[i] = threading.Thread(target=prefix_min, args=(gray,results,i))
+        threads[i].start()
+
+    # stop and join them back
+    for i in range(0,NUM_THREADS):
+        threads[i].join()
+
+
+    #NOTE: try changing the mask to include right->left instead of 
+    # left->right, will it change the crop bias????
+
+    #NOTE: Doesn't matter.....
+
+    combined_mask = get_mask(results=results)
+
+
+    max_area = 0
+    largest_bbox = None
+    largest_contour = None
+
+
+    #NOTE: add another heuristic that eliminates distinctly small rectangles (get area of pipette when detected)
+
+
+    # countour selection heuristics
+    lwr_bound = 3.5
+    high_bound = 4.4
+
+    # area selection
+    area_bound = 1000
+
+    copy_img  = img.copy()
+
+    contours, _ = cv2.findContours(combined_mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    for contour in contours:
+        x, y, w, h = cv2.boundingRect(contour)
+        area = w*h
+
+        ratio = w/h
+
+        if (ratio >= lwr_bound and ratio <= high_bound and area > max_area and area > area_bound):
+            max_area = area
+            largest_bbox = (x,y,w,h)
+            largest_contour = contour
+
+    if largest_bbox is not None:
+        x,y,w,h = largest_bbox
+
+        # print(f"RATION w/h: {w/h}")
+
+        skew_angle = get_skew_angle(largest_contour)
+        # cv2.drawContours(img,largest_contour,-1,(0,255,0),2)
+
+        # print(f"SKEW: {skew_angle} \t CORRECTION: {skew_angle*-1.0}")
+
+        if skew_angle !=  0.0:
+            rotated_img = rotate_image(img,-1.0*skew_angle)
+        else:
+            rotated_img = img
+        cv2.rectangle(copy_img,(x,y),(x+w,y+h),(255,0,0),2)
+        # NOTE: applied heursitic to right and bottom sides
+        heuristic = 5
+
+        cropped_img = rotated_img[y+heuristic:y+h-heuristic, x+heuristic:x+w-heuristic]
+
+    else:
+        cropped_img = img
+
+    # cv2.imshow('bounding box',copy_img)
+
+    return cropped_img, copy_img

computer_vision/number_detection/number_detection.py

@@ -1,53 +1,10 @@
 import cv2
 import pytesseract
 from pytesseract import image_to_string
-import os
 import numpy as np
-import time
-import platform
 import re
 
-from parallelPrefix import crop_image
-
-
-'''
-Noise Removal:
-    - Clustering algorithm based on camera positions
-
-    - Gaussian Blur
-    
-    - hybrid between neural and pretrained
-        - build a layer ontop of pretrained model with a neural network
-        - needs MORE DATA LOTS OF DATA
-    
-    - Find out what EXACTLY our camera will be reading the values
-
-    - Make cropping algo
-'''
-
-# def read_camera(): 
-#     current_os = platform.system()
-
-#     if current_os == "Windows":
-#         cap = cv2.VideoCapture(0, cv2.CAP_DSHOW)  
-#     elif current_os == "Darwin": 
-#         cap = cv2.VideoCapture(0, cv2.CAP_AVFOUNDATION)
-#     elif current_os == "Linux":
-#         cap = cv2.VideoCapture(0, cv2.CAP_V4L2)
-#     else:
-#         cap = cv2.VideoCapture(0)
-
-#     # set lowest point of webcam 5 inches above the surface
-#     while True:
-#         ret, frame = cap.read()
-
-#         if not ret:
-#             break
-
-#         # crop frame first
-#         num_reader(frame)
-#         cv2.imshow("test", frame)
-#         cv2.waitKey(1)
+from crop_tool import crop_image
 
 def preprocessing(img,debug=False,still=False):
     #grayscale

computer_vision/number_detection/parallelPrefix.py

@@ -1,11 +1,8 @@
 import cv2
-import os
-import glob
 import numpy as np
-import timeit
 import threading
-import multiprocessing
 import ctypes
+import platform
 
 '''
 We are going to implement a prefix-minimum solution to the cropping problem
@@ -30,7 +27,19 @@ def init():
     global lib
     # Define the argument and return types of the C function
     try:
-        lib = ctypes.CDLL('./library/number-detection-pkg.dylib')
+        #evaluate platform that's running this file
+        system = platform.system()
+
+        match system:
+            case 'Darwin': #macOS
+                lib = ctypes.CDLL('./library/number-detection-pkg.dylib')
+            case 'Windows':
+                lib = ctypes.CDLL('./library/number-detection-pkg.dll')
+            case 'Linux':
+                lib = ctypes.CDLL('./library/number-detection-pkg.so')
+            case _:
+                raise OSError(f"Unsupported operating system: {system}")
+
         lib.prefix_min.argtypes = [ctypes.POINTER(ctypes.c_uint8), ctypes.POINTER(ctypes.c_uint8), ctypes.c_int, ctypes.c_int, ctypes.c_int]
         lib.prefix_min.restype = None
         _initialized = True

computer_vision/number_detection/test-crop.py → computer_vision/number_detection/test_crop.py

@@ -4,9 +4,7 @@
 import glob
 import argparse
 import time
-from parallelPrefix import crop_image
-
-
+from crop_tool import crop_image
 
 class TestCropping(unittest.TestCase):
 
@@ -62,7 +60,7 @@ def test_all(self):
         return
 
 #NOTE: change testAll if you want to test all files or not
-def main(testAll=True):
+def main(testAll=False):
     # user should give us an image file arg
     suite = unittest.TestSuite()
     if testAll:

computer_vision/number_detection/test-number-detection.py → computer_vision/number_detection/test_number_detection.py

@@ -8,30 +8,6 @@
 filePath = './test/cropped/'
 txtPath = './test/testResults/'
 
-
-
-'''
-TEST NOTES:
-    - Two variations, one that runs on all files in cropped and another
-    where the user passes in a single filename
-
-    - Results are stored in testResults and should show you which files failed any why.
-    Including a total tally and correct ratio.
-
-    NOTE: add a heuristic to cropping to get rid of any white on edges from left side
-
-    NOTE: find a way to deal with dial not being fully turned edge case
-    19.5.jpg        FAILED! RESULT: 49.5     EXPECTED: 19.5 
-    02.5.jpg        FAILED! RESULT: 02       EXPECTED: 02.5 
-    07.0.jpg        FAILED! RESULT: 2..0     EXPECTED: 07.0 
-    09.5.jpg        FAILED! RESULT: 09       EXPECTED: 09.5 
-    15.5.jpg        FAILED! RESULT: 16       EXPECTED: 15.5 
-    11.0.jpg        FAILED! RESULT: 7112.0   EXPECTED: 11.0 
-    11.5.jpg        FAILED! RESULT: 11.0     EXPECTED: 11.5 
-    17.0.jpg        FAILED! RESULT: 172.0    EXPECTED: 17.0 
-
-'''
-
 class TestNumberDetection(unittest.TestCase):
     def test_function_multiple_times(self):
         # go through each file in the cropped directory