-
Notifications
You must be signed in to change notification settings - Fork 8
/
helper_functions.py
154 lines (137 loc) · 5.79 KB
/
helper_functions.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
import numpy as np
import cv2
import os
from scipy.io import loadmat
from shutil import copyfile
# filedir = ("/homes/mat10/Documents/MSc_Machine_Learning/"
# "ISO_Deep_Lip_Reading/Stafylakis_Tzimiropoulos/"
# "Tensorflow_Implementation/frames/mouths2")
# subdirs = ['word1', 'word2']
# files = "all"
# if files == 'all':
# files = [[file for file in os.listdir(filedir+subdir) if file[-3:] == 'jpg'] for subdir in subdirs]
def get_npy_data(paths):
"""
imports data and stacks them in the 3rd dimension
:return:
"""
data = np.array([np.load(path) for path in paths])
data = data[:, :, 3:115, 3:115]
nbatch, nframes, width, height = data.shape
data = data.reshape(nbatch, nframes, width, height, 1)
return data
def get_data(filedir, subdirs, files='all'):
"""
imports data and stacks them in the 3rd dimension
:return:
"""
img = [[cv2.imread(make_path(filedir, subdir, file), 0)
for file in os.listdir(make_path(filedir, subdir)) if file[-3:] == 'jpg']
for subdir in subdirs]
img = np.array(img)
nbatches, n, w, h = img.shape
return img.reshape(nbatches, n, w, h, 1)
# old function
# def get_data(filedir, subdirs, files='all'):
# """
# imports data and stacks them in the 3rd dimension
# :return:
# """
# img = [[cv2.imread(make_path(filedir, subdir, file), 0)
# for file in os.listdir(make_path(filedir, subdir)) if file[-3:] == 'jpg']
# for subdir in subdirs]
# img = np.array(img)
# nbatches, n, w, h = img.shape
# return img.reshape(nbatches, n, w, h, 1)
def make_path(*args):
last_arg = args[-1]
args = [arg + "/" for arg in args[:-1]]
path = ''.join(args) + last_arg
return path
def video2frames(filedir, file, savedir):
vidcap = cv2.VideoCapture(filedir + file)
success, image = vidcap.read()
count = 0
success = True
while success:
success, image = vidcap.read()
print('Read a new frame: ', success)
cv2.imwrite(savedir + "%s_frame%d.jpg" % (file[:-5], count), image) # save frame as JPEG file count += 1
count += 1
print('Saved %d frames' %count)
def extract_mouth_roi(files_dir, dataset, word, file, lms_dir, resolution=118,
save_file_name=None):
# facial Landmarks
lms_file = 'lipread_mp4__%s__%s__%s.mat' % (word, dataset, file[:-4])
lms = loadmat(lms_dir + "/" + lms_file)
lms = lms['pts']
# if lms.shape[0] != 136: # LOG if lms.shape[0] != 136
# print()
lms_x = lms[:68, :]
lms_y = lms[68:, :]
mouth_center_x = int(np.median(np.mean(lms_x[48:68, :], axis=1)))
mouth_center_y = int(np.median(np.mean(lms_y[48:68, :], axis=1)))
# video
vidcap = cv2.VideoCapture(files_dir + file)
success, image = vidcap.read()
gray_image = [cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)]
count = 1
while success:
success, image = vidcap.read()
# print('Read a new frame: ', success)
if success:
gray_image.append(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))
count += 1
# cv2.imwrite(savedir + "%s_frame%d.jpg" % (file[:-5], count), image) # save frame as JPEG file count += 1
if count != 29:
print("File %s has %d frames" % (files_dir + file, count))
gray_image = np.array(gray_image)
# mouth roi
# dimensions of square mouth region, 112 +- 6 pixels for data augmentation later
dim = resolution // 2
mouth_roi = gray_image[:, mouth_center_y - dim : mouth_center_y + dim,
mouth_center_x - dim : mouth_center_x + dim]
# either save on disk or return array
if save_file_name:
np.save(save_file_name, mouth_roi)
else:
return mouth_roi
# def save_data_from_videos(data_root_dir, lms_dir, save_root_dir, words='all', datasets='all'):
# if words == "all":
# words = os.listdir(data_root_dir) # list of words
# if datasets == "all":
# datasets = ['train', 'val', 'test']
# for dataset in datasets: # train, test, val
# for word in words: # ABOUT, ...
# print(dataset + "-" + word)
# # check if directory already exists
# if not os.path.isdir(save_root_dir + "/" + word):
# os.makedirs(save_root_dir + "/" + word)
# if not os.path.isdir(save_root_dir + "/" + word + "/" + dataset):
# os.makedirs(save_root_dir + "/" + word + "/" + dataset)
# files_dir = data_root_dir + "/" + word + "/" + dataset + "/"
# files_mp4 = [file for file in os.listdir(files_dir) if file[-3:] == 'mp4']
# files_txt = [file for file in os.listdir(files_dir) if file[-3:] == 'txt']
# # file = files_mp4[0]
# for i, file in enumerate(files_mp4):
# save_dir = save_root_dir + "/" + word + "/" + dataset + "/"
# # check if save directory already exists
# # if not os.path.isdir(save_dir):
# # os.makedirs(save_dir)
# # copy txt file to save directory
# file_txt = files_txt[i]
# copyfile(files_dir + file_txt, save_dir + file_txt)
# # split mp4 to frames and save to save directory
# # video2frames(files_dir, file, save_dir)
# save_file_name = save_root_dir + "/" + word + "/" + dataset + "/" + file[:-4] + ".npy"
# extract_mouth_roi(files_dir, dataset, word, file, lms_dir,
# resolution=118, save_file_name=save_file_name)
# def get_data():
# """
# imports data and stacks them in the 3rd dimension
# :return:
# """
# img = [cv2.imread(filedir + file, 0) for file in files]
# img = np.array(img)
# n, w, h = img.shape
# return img.reshape(1, n, w, h, 1)