Everything

Author: Vaibhav

Paper.pdf

@@ -0,0 +1,37 @@
+# VideoStabilization 
+
+## Description
+
+This is a python-openCV implementation of this [research paper](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/37041.pdf) by Matthias Grundmann, Vivek Kwatra and Irfan Essa. The basic goal of this piece of code is to do some PreProcessing on a given wobbly video, and then output a relatively stable video which resembles professional videography(without bumps).
+
+## Basic Algorithm and Working
+
+### Transforms Extraction(achieved by preproc.py)
+
+Firstly, the affine transform between all the consecutive 2 frames are computed, so that we achieve an estimate of the camera trajectory.
+
+### Stabilizing(achieved by stabilize.py)
+
+Now, the above camera trajectory will be wobbly and needs to be smoothened. This is achieved by converting the problem to an optimization problem where the derivative needs to be minimized and the constraint is that the values don't differ more than a threshold from the original ones.
+
+### Video Output(achieved by generate.py)
+
+Finally a cropped region of the video is outputted where the cropped region box follows a trajectory that smoothens the output.
+
+## How To
+
+* Run the webapp: 	
+```console
+	bar@foo:~/VideoStabilization$ python3 preproc.py <video_file>
+	bar@foo:~/VideoStabilization$ python3 stabilize.py
+	bar@foo:~/VideoStabilization$ python3 generate.py <video_file>	
+```
+
+## Built With
+
+* [Python3](https://www.python.org/download/releases/3.0/)
+* [OpenCV](https://docs.opencv.org/)
+
+## Author
+
+* Vaibhav Garg

Readme.md

@@ -0,0 +1,64 @@
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import pickle
+import sys
+
+# Loading the data
+with open('transforms.pkl', 'rb') as f:
+	transforms = pickle.load(f)
+
+with open('trajectory.pkl', 'rb') as f:
+    trajectory = pickle.load(f)
+
+with open('smoothTrajectory.pkl', 'rb') as f:
+    smoothTrajectory = pickle.load(f)
+
+difference = trajectory - smoothTrajectory
+
+# Creating output video
+v = cv2.VideoCapture(sys.argv[1])
+W = int(v.get(cv2.CAP_PROP_FRAME_WIDTH))
+H = int(v.get(cv2.CAP_PROP_FRAME_HEIGHT))
+fps = int(v.get(cv2.CAP_PROP_FPS))
+DispThresh = 50	
+DispThresh += 10
+
+curX = DispThresh
+curY = DispThresh
+
+out = cv2.VideoWriter('video_out.avi', cv2.VideoWriter_fourcc(*'XVID'), fps, (W-2*DispThresh, H-2*DispThresh))
+count = 0 
+m = np.zeros([2, 3])
+
+while(v.isOpened()):
+    ret, frame = v.read()
+    if ret == True:
+        if count > 0:
+        	curX = DispThresh+difference[0][count-1]
+        	curY = DispThresh+difference[1][count-1]
+        
+        # Writing to output file
+        boxPart = frame[np.int(curY):np.int(curY)+H-2*DispThresh, np.int(curX):np.int(curX)+W-2*DispThresh, :]
+        out.write(boxPart)
+                
+        # Displaying the difference
+        cv2.rectangle(frame, (np.int(curX), np.int(curY)), (np.int(curX)+W-2*DispThresh, np.int(curY)+H-2*DispThresh), (0, 255, 0), 3)
+        frame = cv2.resize(frame, (W-2*DispThresh, H-2*DispThresh))
+        dispFrame = cv2.hconcat([frame, boxPart])
+        if dispFrame.shape[1] > 1920:
+        	dispFrame = cv2.resize(dispFrame, (dispFrame.shape[1]//2, dispFrame.shape[0]//2));
+        cv2.imshow('DispFrame', dispFrame)
+        
+        count +=1
+        if count == 1000:
+            break
+
+        if cv2.waitKey(20) & 0xFF == ord('q'):
+        	break
+    else:
+        break
+
+# When everything done, release the capture
+v.release()
+out.release()

generate.py

@@ -0,0 +1,61 @@
+import cv2
+import numpy as np
+import pickle
+import sys
+
+sift = cv2.xfeatures2d.SIFT_create()
+def getAffMat(I1, I2):
+    I1 = cv2.cvtColor(I1, cv2.COLOR_BGR2GRAY)
+    I2 = cv2.cvtColor(I2, cv2.COLOR_BGR2GRAY)
+
+    # Finding sift features
+    kp1, desc1 = sift.detectAndCompute(I1, None)
+    kp2, desc2 = sift.detectAndCompute(I2, None)
+
+    # Finding good matches using ratio testing
+    bf = cv2.BFMatcher()
+    matches = bf.knnMatch(desc1, desc2, k=2)
+
+    good = []
+    for m,n in matches:
+        if m.distance < 0.7*n.distance:
+            good.append(m)
+
+    pts_src = []
+    pts_dst = []
+    for i in range(len(good)):
+    	pts_src.append([kp1[good[i].queryIdx].pt[0], kp1[good[i].queryIdx].pt[1]])
+    	pts_dst.append([kp2[good[i].trainIdx].pt[0], kp2[good[i].trainIdx].pt[1]])
+
+    pts_src = np.array(pts_src).astype(np.float32)
+    pts_dst = np.array(pts_dst).astype(np.float32)
+
+    # Computing affine matrix using the best matches
+    return cv2.estimateRigidTransform(pts_src, pts_dst, fullAffine=False)
+
+v = cv2.VideoCapture(sys.argv[1])
+
+# Generating the Xdata and Ydata
+transforms = [[], []]
+count = 0
+while v.isOpened():
+    ret, frame = v.read()
+    if ret == True:
+    	if count > 0:
+    		transMat = getAffMat(prev, frame)
+    		transforms[0].append(transMat[0][2])
+    		transforms[1].append(transMat[1][2])
+    		
+    	count += 1
+    	prev = frame
+    	print(count)
+    	if count == 1000:
+    		break
+    else:
+    	break
+
+v.release()
+
+# Storing the data
+with open('transforms.pkl', 'wb') as f:
+	pickle.dump(transforms, f)

preproc.py

@@ -0,0 +1,63 @@
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import pickle
+import cvxpy as cp
+
+# Loading the data
+with open('transforms.pkl', 'rb') as f:
+	transforms = pickle.load(f)
+
+# Computing the trajectory
+trajectory = np.cumsum(transforms, axis=1)
+with open('trajectory.pkl', 'wb') as f:
+	pickle.dump(trajectory, f)
+
+
+# Smoothening the trajectories
+# fx is the optimal x trajectory
+# fy is the optimal y trajectory
+fx = cp.Variable(len(trajectory[0]))
+fy = cp.Variable(len(trajectory[1]))
+
+lbd1 = 1000
+lbd2 = 100
+lbd3 = 10000
+DispThresh = 50
+constraints = [cp.abs(fx - trajectory[0]) <= DispThresh,
+ 			   cp.abs(fy - trajectory[1]) <= DispThresh]
+
+# Defining the minimization objective function
+obj = 0																																																																																								
+for i in range(len(trajectory[0])):
+	obj += ( (trajectory[0][i]-fx[i])**2 + (trajectory[1][i]-fy[i])**2 )
+
+# DP1
+for i in range(len(trajectory[0])-1):
+	obj += lbd1*(cp.abs(fx[i+1]-fx[i]) + cp.abs(fy[i+1]-fy[i]))
+
+# DP2
+for i in range(len(trajectory[0])-2):
+	obj += lbd2*(cp.abs(fx[i+2]-2*fx[i+1]+fx[i]) + cp.abs(fy[i+2]-2*fy[i+1]+fy[i]))
+
+# DP3
+for i in range(len(trajectory[0])-3):
+	obj += lbd3*(cp.abs(fx[i+3]-3*fx[i+2] + 3*fx[i+1]-fx[i]) + cp.abs(fy[i+3]-3*fy[i+2]+3*fy[i+1]-fy[i]))
+
+prob = cp.Problem(cp.Minimize(obj), constraints)
+prob.solve()
+
+# Results
+plt.subplot(1, 2, 1)
+plt.plot(trajectory[0])
+plt.plot(fx.value)
+plt.subplot(1, 2, 2)
+plt.plot(trajectory[1])
+plt.plot(fy.value)
+plt.show()
+
+smoothTrajectory = np.array([fx.value, fy.value])
+
+# Storing smooth trajectory
+with open('smoothTrajectory.pkl', 'wb') as f:
+	pickle.dump(smoothTrajectory, f)