Repository navigation
Expand file tree
/
Copy pathdata.py
More file actions
198 lines (159 loc) · 6.68 KB
/
Copy pathdata.py
File metadata and controls
198 lines (159 loc) · 6.68 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
'''This module provides all the methods needed for data extraction, preprocessing, storing and retrieval.
A deep neural network is nothing but a bunch of random parameters without massive amounts of high quality data
to train it on, and as such a large percentage of project time was spent building the data.py methods. Our main
storage system is HDF5 as it provides a pythonic object oriented approach to storing along with numpy combability and
very fast streaming capabilities. An additional plus was the ability of "fancy indexing" or list of indexes type of
sample access. Most of the details are provided in the methods docstrings
'''
from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
import os
import pickle
import random
import cv2
import numpy as np
import h5py
from keras.utils import np_utils
from workspace import DEFAULT_WORKSPACE
def extract_data(workspace=DEFAULT_WORKSPACE, size=256):
'''Extracts the data from the downloaded_images folders
Attributes:
size: The size to which to resize the images. All images must be the same size so that
they can be trained using a Deep CNN.
e.g size=256, images will be 256x256
Returns a list(X) with the images and their labels(y) in string form, e.g.('dog')
'''
X = []
y = []
count = 0
for subdir, dir, files in os.walk(workspace.raw_images_dir):
for file in files:
if file.endswith(".jpg") or file.endswith(".png") or file.endswith(".jpeg"):
bits = subdir.split("/")
category = bits[-1]
filepath = os.path.join(subdir, file)
image = cv2.imread(filepath)
try:
image = resize(image, size=size)
if image is not None:
X.append(image)
y.append(category)
count = count + 1
print("Images proccesed {0}".format(count))
except:
pass
return X, y
def preprocess_data(X, y, workspace=DEFAULT_WORKSPACE, save=True, preset=None, subtract_mean=True):
'''Preprocesses the data that have already been extracted with extract_data() method
Attributes:
X: A list containing images of shape (3, width, height)
y: A list containing the image labels in string form
Returns data in numpy form ready to be used for training
'''
X = np.array(X)
categories = set()
for label in y: # get all the unique categories
categories.add(label)
# build a dictionary that maps categories in string form to a unique id
categories = dict(zip(categories, list(range(len(categories)))))
y_temp = []
for label in y: # encode y with the unique ids
y_temp.append(categories[label])
y_temp = np.array(y_temp)
X = X.astype(np.float32)
mean = X.mean(axis=0) # get mean
if subtract_mean:
X = X - mean
# save categories for future use
pickle.dump(categories, open(workspace.categories_path, "wb"))
y = np_utils.to_categorical(y_temp, max(y_temp) + 1)
if preset is not None:
y = np_utils.to_categorical(y_temp, preset)
n_samples = y.shape[0]
n_categories = len(categories)
if save:
with h5py.File(workspace.data_path, 'w') as h5f:
h5f.create_dataset('X', data=X)
h5f.create_dataset('y', data=y)
h5f.create_dataset('nb_samples', data=n_samples)
h5f.create_dataset('n_categories', data=n_categories)
h5f.create_dataset('mean', data=mean)
return X, y, n_samples, len(categories)
def get_mean(workspace=DEFAULT_WORKSPACE):
try:
with h5py.File(workspace.dataset_path, 'r') as h5f:
return h5f['mean'][:]
except IOError:
return np.load(workspace.mean_path)
def get_categories(workspace=DEFAULT_WORKSPACE):
'''Load categories names'''
categories = pickle.load(open(workspace.categories_path, "rb"))
return categories
def produce_train_indices(dataset_indx, number_of_samples, val_indx):
dataset_indx = np.delete(dataset_indx, val_indx)
np.random.seed(seed=None)
train = np.random.choice(dataset_indx, size=number_of_samples)
train = np.unique(train)
return (np.sort(train)).tolist()
def produce_validation_indices(dataset_indx, number_of_samples):
np.random.seed(2048)
val = np.random.choice(dataset_indx, size=number_of_samples)
val = np.sort(val)
val = np.unique(val)
return val.tolist()
def load_dataset_bit_from_hdf5(workspace, train_indices, val_indices, only_train=True):
if only_train:
with h5py.File(workspace.data_path, 'r') as h5f:
X_train = f['X'][train_indices]
y_train = f['y'][train_indices]
return X_train, y_train
else:
with h5py.File(workspace.data_path, 'r') as h5f:
X_train = h5f['X'][train_indices]
y_train = h5f['y'][train_indices]
X_val = h5f['X'][val_indices]
y_val = h5f['y'][val_indices]
return X_train, y_train, X_val, y_val
def augment_data(X_train, random_angle_max=180, mirroring_probability=0.5):
'''Augment data with random mirrors and random rotations'''
for i in range(len(X_train)):
random_angle = random.randint(0, random_angle_max)
mirror_decision = random.randint(0, 100)
flip_orientation = random.randint(0, 1)
for channel in range(len(X_train[i])):
rows, cols = X_train[i, channel].shape
M = cv2.getRotationMatrix2D((cols // 2, rows // 2), random_angle, 1)
rotated_image = cv2.warpAffine(X_train[i, channel], M, (cols, rows))
if mirror_decision < mirroring_probability * 100:
X_train[i, channel] = cv2.flip(rotated_image, flipCode=flip_orientation)
else:
X_train[i, channel] = rotated_image
return X_train
def resize(img, size):
"""resize image into size x size
Attributes:
img: The image to resize
size: The size to resize the image
"""
print("preresize")
img = cv2.resize(img, (size, size))
print("preroll")
img = np.rollaxis(img, 2)
return img
def split_data(X, y, split_ratio=0.5):
'''Splits data into training and testing-sets'''
random.seed(4096)
train_idx = []
test_idx = []
for i in range(X.shape[0]):
decision = random.randint(0, 99)
if decision < (100 - (split_ratio * 100)):
train_idx.append(i)
else:
test_idx.append(i)
X_train = X[train_idx]
X_test = X[test_idx]
y_train = y[train_idx]
y_test = y[test_idx]
return X_train, y_train, X_test, y_test