Project 2 Submission

convolution.py

@@ -0,0 +1,158 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from PIL import Image
+import sys
+
+# convolve image g with kernel h
+def convolve(g, h, std=False, color_channel=False):
+    g = g.astype(dtype='float32')
+
+    # image dimensions and color channels (add color channel if grayscale)
+    g_shape = None
+    if (len(g.shape) == 2):
+        g_shape = g.shape
+        height, width = g.shape
+        g = g.reshape((height, width, 1))
+    height, width, colors = g.shape
+    g_h = np.zeros_like(g)
+
+    # size of neighborhood
+    size = (int)((h.shape[0]-1)/2)
+
+    # iterate each color band
+    for color in range(colors):
+        # iterate each (u,v) pixel (offset 1 from edges)
+        for u in range(size, height-size):
+            for v in range(size, width-size):
+                g_h[u,v,color] = convolve_pixel(g, h, size, u, v, color)
+
+    if std == True:
+        standardize_color_channel(g, width, height, colors, size)
+
+    # reshape to grayscale if necessary
+    if (g_shape != None):
+        g_h = g_h.reshape(g_shape)
+
+    return g_h
+
+# find the convolution of (g*h)(u,v) at pixel [u,v,color]
+def convolve_pixel(g, h, size, u, v, color):
+    g_neighborhood = g[u-size:u+size+1, v-size:v+size+1, color]
+    g_h = 0
+
+    # convolve over 3x3 neighborhood
+    g_h = np.multiply(g_neighborhood,h)
+    g_h = g_h.sum()
+
+    return g_h
+
+# standardize color channels over range (0,255)
+def standardize_color_channel(g, width, height, colors, size):
+    # highest and lowest intensity
+    I_max = None
+    I_min = None
+    for color in range(colors):
+        for u in range(size, height-size):
+            for v in range(size, width-size):
+                if I_min == None or g[u,v,color] < I_min:
+                    I_min = g[u,v,color]
+                if  I_max == None or g[u,v,color] > I_max:
+                    I_max = g[u,v,color]
+    I_range = I_max - I_min
+    for color in range(colors):
+        for u in range(1, height-1):
+            for v in range(1, width-1):
+                g[u,v,color] = (255 / I_range) * (g[u,v,color] + I_min)
+    return g
+
+# Linear (Box) Blur kernel
+def linear_kernel(size):
+    n = 1+2*size
+    h = np.zeros((n, n))
+
+    for j in range(-size, size+1):
+        for k in range(-size, size+1):
+            h[j+size,k+size] = 1
+
+    # Calculate Unity constant
+    Z = 1/np.sum(h)
+    h = Z * h
+
+    return h
+
+# Gaussian Blur kernel
+def gaussian_kernel(sigma, size):
+    n = 1+2*size
+    h = np.zeros((n, n))
+
+    for j in range(-size, size+1):
+        for k in range(-size, size+1):
+            h[j+size,k+size] = np.exp(-1 * (np.power(j,2) + np.power(k,2)) / (2 * np.power(sigma,2)))
+
+    # Calculate Unity constant
+    Z = 1/np.sum(h)
+    h = Z * h
+
+    return h
+
+# Sobel Operator kernel (u), for Edge Detection
+def sobel_u_kernel():
+    h = np.array([[-1, 0, 1],
+                  [-2, 0, 2],
+                  [-1, 0, 1]])
+    return h
+
+# Sobel Operator kernel (v), for Edge Detection
+def sobel_v_kernel():
+    h = np.array([[-1, -2, -1],
+                  [ 0,  0,  0],
+                  [ 1,  2,  1]])
+    return h
+
+# convert image to grayscale using "perceptual luminance-preserving formula"
+def rgb2gray(rgb):
+    return np.dot(rgb[...,:3], [0.299, 0.587, 0.114])
+
+#############################################################################
+##############################       MAIN     ###############################
+#############################################################################
+
+# # import image
+# img_color = plt.imread('noisy_big_chief.jpeg')
+# img_gray = img_color.mean(axis=2)
+#
+# # show both image
+# # fig, ax = plt.subplots(1, figsize=(12,8))
+# # plt.imshow(img_color)
+# # plt.show()
+# # plt.imshow(img_gray, cmap='gray')
+# # plt.show()
+# # plt.imsave('bnc_gray.jpeg', img_gray, cmap='gray')
+#
+# # Linear Blur/Smoothing
+# h_linear = linear_kernel(5)
+# img_linear = convolve(img_color, h_linear).astype(dtype='uint8')
+# plt.imshow(img_linear)
+# plt.show()
+# plt.imsave('bnc_linear.jpeg', img_linear)
+#
+# # Gaussian Blur/Smoothing
+# h_gaussian = gaussian_kernel(3, 5)
+# img_gaussian = convolve(img_color, h_gaussian).astype(dtype='uint8')
+# plt.imshow(img_gaussian)
+# plt.show()
+# plt.imsave('bnc_gaussian.jpeg', img_gaussian)
+#
+# # Sobel-u Edge Detection
+# h_sobelu = sobel_u_kernel()
+# img_sobelu = convolve(img_gray, h_sobelu)
+# plt.imshow(img_sobelu, cmap='gray')
+# plt.show()
+# plt.imsave('bnc_sobelu.jpeg', img_sobelu, cmap='gray')
+#
+# # Sobel-v Edge Detection
+# h_sobelv = sobel_v_kernel()
+# img_sobelv = convolve(img_gray, h_sobelv)
+# plt.imshow(img_sobelv, cmap='gray')
+# plt.show()
+# plt.imsave('bnc_sobelv.jpeg', img_sobelv, cmap='gray')

keypoint_detector.py

@@ -0,0 +1,37 @@
+import numpy as np
+import matplotlib.pyplot as plt
+from PIL import Image
+import sys
+
+# local imports
+from convolution import *
+
+# find local maxima of Harris Response matrix
+def local_maxima(g, h, size):
+    return 0
+
+# Check if a given pixel is a local maxima
+def check_if_local_maxima(H, size, width, height, u, v):
+    # test pixel Harris score
+    px = H[u,v]
+    # iterate over neighborhood
+    for j in range(u-size, u+size+1):
+        for k in range(v-size, v+size+1):
+            if j<0 or k<0 or j>height-1 or k>width-1:
+                continue
+            # skip itself
+            if j==u and k==v:
+                continue
+            # if pixel of greater score found, not maxima
+            elif px < H[j,k]:
+                return False
+    # else, is maxima
+    return True
+
+def sum_sq_error(p1, p2):
+	sum = 0
+
+	for i, j in zip(p1, p2):
+		sum += (i - j)**2
+
+	return (sum)

project_description.ipynb

@@ -90,7 +90,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.7.1"
   }
  },
  "nbformat": 4,

stitcher.py

@@ -0,0 +1,148 @@
+import numpy as np
+import matplotlib.pyplot as plt
+
+from keypoint_detector import *
+
+class Stitcher(object):
+	def __init__(self,image_1,image_2):
+		self.images = [image_1,image_2]
+		self.images[0] = self.images[0].mean(axis=2)
+		self.images[1] = self.images[1].mean(axis=2)
+
+	def find_keypoints(self, img):
+		"""
+		Step 1: This method locates features that are "good" for matching.  To do this we will implement the Harris 
+		corner detector
+		"""
+		# import image and convert to grayscale
+		I = img
+
+		# detect_keypoints(I)
+		g = I
+
+		h_sobelu = sobel_u_kernel()
+		h_sobelv = sobel_v_kernel()
+		h_gaussian = gaussian_kernel(2, 2)
+
+		g_u = convolve(g, h_sobelu)
+		g_u2 = np.multiply(g_u, g_u)
+		g_u2 = convolve(g_u2, h_gaussian)
+
+		g_v = convolve(g, h_sobelv)
+		g_v2 = np.multiply(g_v, g_v)
+		g_v2 = convolve(g_v2, h_gaussian)
+
+		g_uv = np.multiply(g_u, g_v)
+		g_uv = convolve(g_uv, h_gaussian)
+
+		# Harris Response matrix
+		H = np.zeros_like(g)
+		# Determinant(g)
+		H_det = np.multiply(g_u2, g_v2) - np.multiply(g_uv, g_uv)
+		# Trace(g) (small number to prevent division by zero)
+		H_tr = g_u2 + g_v2 + 1e-10
+		# H = det(g)/tr(g)
+		H = np.divide(H_det, H_tr)
+
+		# Find local maxima of Harris matrix scores
+		height, width = H.shape
+		max = []
+		# size of neighborhood
+		size = 50
+
+		# iterate over pixels
+		for u in range(height):
+			for v in range(width):
+				# add to list if local maxima
+				if check_if_local_maxima(H, size, width, height, u, v):
+					max.append([v,u])
+
+		for i in range(len(max)):
+			plt.scatter(x=max[i][0], y=max[i][1], c='r')
+
+		plt.imshow(I,cmap=plt.cm.gray)
+		plt.show()
+
+		return max
+
+	def generate_descriptors(self,img,points, l):
+		"""
+		Step 2: After identifying relevant keypoints, we need to come up with a quantitative description of the 
+		neighborhood of that keypoint, so that we can match it to keypoints in other images.
+		"""
+		l = int(l/2)
+
+		descriptors = []
+		for i in range(len(points)):
+			x1 = points[i][0]-l
+			x2 = points[i][0]+l+1
+			y1 = points[i][1]-l
+			y2 = points[i][1]+l+1
+
+			if(x1 < 0):
+				x1 = 0
+			if(y1 < 0):
+				y1 = 0
+			if(x2 > img.shape[0]):
+				x2 = img.shape[0]
+			if(y2 > img.shape[1]):
+				y2 = img.shape[1]
+
+			descriptors.append([img[x1:x2:1, y1:y2:1], points[i]]) #Returns an array with [actual img data, img location (u,v)]
+		return descriptors
+
+	def match_keypoints(self):
+		"""
+		Step 3: Compare keypoint descriptions between images, identify potential matches, and filter likely
+		mismatches
+		"""
+		matches = []
+		m1 = myStitcher.find_keypoints(self.images[0])
+		m2 = myStitcher.find_keypoints(self.images[1])
+		desc1 = myStitcher.generate_descriptors(self.images[0], m1, 21)
+		desc2 = myStitcher.generate_descriptors(self.images[1], m2, 21)
+
+		for p1 in desc1:
+			x1, y1 = p1[1][0], p1[1][1]
+			errors = []
+			for p2 in desc2:
+				x2, y2 = p2[1][0], p2[1][1]
+				errors.append([[x2, y2], sum_sq_error(p1[0].flatten(), p2[0].flatten())])
+			errors.sort(key=lambda x: x[1])
+			matches.append([[x1, y1], errors[0][0]])
+		return matches
+
+	def find_homography(self):
+		"""
+		Step 4: Find a linear transformation (of various complexities) that maps pixels from the second image to 
+		pixels in the first image
+		"""
+
+	def stitch(self):
+		"""
+		Step 5: Transform second image into local coordinate system of first image, and (perhaps) perform blending
+		to avoid obvious seams between images.
+		"""
+
+im1 = plt.imread('class_photo1re.jpg')
+im2 = plt.imread('class_photo2re.jpg')
+myStitcher = Stitcher(im1, im2)
+matches = myStitcher.match_keypoints()
+
+I = np.concatenate((im1, im2), axis=1)
+
+plt.imshow(I, cmap=plt.cm.gray)
+
+offset = im1.shape[1]
+lengthOfMatches = len(matches)
+
+x1 = [matches[i][0][0] for i in range(lengthOfMatches)]
+y1 = [matches[i][0][1] for i in range(lengthOfMatches)]
+
+x2 = [matches[i][1][0] + offset for i in range(lengthOfMatches)]
+y2 = [matches[i][1][1] for i in range(lengthOfMatches)]
+
+for i in range(lengthOfMatches):
+	plt.plot([x1[i], x2[i]], [y1[i], y2[i]])
+
+plt.show()