Repository navigation
Expand file tree
/
Copy pathtrain.py
More file actions
140 lines (112 loc) · 6.37 KB
/
Copy pathtrain.py
File metadata and controls
140 lines (112 loc) · 6.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
'''
This module provides all the methods needed to train and validation test a deep neural network. Training includes:
1. Data Extraction or Loading already extracted data
2. Preprocessing:
a. Mean subtraction + pixel scaling + storing of dataset mean for use in deployment.
b. Support for large datasets via HDF5 real-time sampling during training. The sample is randomized for training set
and deterministic for validation set. By choosing a percentage of your training set of say 0.2 you will get 20% of
your data at random for each epoch, so if you have 100 000 images you will have in memory 20 000 of them, which is
feasible given today's standard desktops of 16-32 GBs of ram. Any more than that and you'll get out of memory errors
3. Data Augmentation: To increase the robustness of the model against rotated images, mirrored images and in general
to improve the robustness against variability of image poses angles we have implemented data augmented training,
which allows one to randomly rotate and mirror their images in between epochs. This simple technique has proven
invaluable to modern machine learning. Not only does it improve robustness but provides additional samples in the
form of randomly edited original images that helps prevent overfitting, produces a higher accuracy/lower loss and as
a result improves generalization performance.
4. Automatic checkpointing (saving of models) after each epoch (named latest_model_weights). And automatic selection of
best model and saving of it's weights named best_model_weights. As well as support for resuming training from the last
checkpoint using the "cont" argument in the run method. Just run python train.py run continue and you should be up and
running.
5. Validation checks after each epoch, as well as output of top-k accuracy for your convenience.
'''
from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
import argparse
import h5py
import numpy as np
import data
import model as m
def get_top_n_error(preds, y, n):
index_of_true = np.argmax(y, axis=1)
index_of_preds = np.argsort(preds, axis=1)
correct = 0
for i in range(len(index_of_true)):
for j in range(1, n + 1):
if index_of_true[i] == index_of_preds[i, -j]:
correct = correct + 1
break
total = len(y)
accuracy = correct / total
return accuracy
def run(epochs=500, training_percentage=0.4, validation_percentage=0.1, extract=True, cont=True, size=256, top_k=5):
'''Does the routine required to get the data, put them in needed format and start training the model
saves weights whenever the model produces a better test result and keeps track of the best loss'''
if extract:
print("Extracting data..")
X, y = data.extract_data(size=size)
print("Preprocessing data..")
X, y, nb_samples, num_categories = data.preprocess_data(X, y, save=True, subtract_mean=True)
else:
print("Loading data..")
h5f = h5py.File('data.hdf5', 'r')
nb_samples = h5f['nb_samples'].value
num_categories = h5f['n_categories'].value
h5f.close()
print("Number of categories: {}".format(num_categories))
print("Number of samples {}".format(nb_samples))
data_ids = np.arange(start=0, stop=nb_samples)
val_ids = data.produce_validation_indices(data_ids, nb_samples * validation_percentage)
train_ids = data.produce_train_indices(dataset_indx=data_ids, number_of_samples=nb_samples * training_percentage,
val_indx=val_ids)
# X_train, y_train, X_test, y_test = data.split_data(X, y, split_ratio=split)
X_train, y_train, X_val, y_val = data.load_dataset_bit_from_hdf5(train_ids, val_ids, only_train=False)
X_val = X_val / 255
print("Building and Compiling model..")
model = m.get_model(n_outputs=num_categories, input_size=size)
if cont:
# model.load_weights_until_layer("pre_trained_weights/latest_model_weights.hdf5", 26)
model.load_weights("pre_trained_weights/latest_model_weights.hdf5")
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=["accuracy"])
print("Training..")
best_performance = np.inf
for i in range(epochs):
train_ids = data.produce_train_indices(dataset_indx=data_ids, number_of_samples=15000, val_indx=val_ids)
X_train, y_train = data.load_dataset_bit_from_hdf5(train_ids, val_ids, only_train=True)
X_train = X_train / 255
X_train = data.augment_data(X_train)
# fit the model on the batches generated by datagen.flow()
metadata = model.fit(X_train, y_train, validation_data=[X_val, y_val], batch_size=64,
nb_epoch=1, verbose=1, shuffle=True, class_weight=None,
sample_weight=None)
current_loss = metadata.history['loss'][-1]
current_val_loss = metadata.history['val_loss'][-1]
preds = model.predict_proba(X_val, batch_size=64)
print("Loss: {}".format(current_loss))
print("Val_loss: {}".format(current_val_loss))
top_3_error = get_top_n_error(preds, y_val, top_k)
print("Top 3 error: {}".format(top_3_error))
if current_val_loss < best_performance:
model.save_weights("pre_trained_weights/model_weights.hdf5", overwrite=True)
best_performance = current_val_loss
print("Saving weights..")
model.save_weights("pre_trained_weights/latest_model_weights.hdf5", overwrite=True)
def extract_data(size=256):
print("Extracting data..")
X, y = data.extract_data(size=256)
print("Preprocessing data..")
X, y, nb_samples, num_categories = data.preprocess_data(X, y, save=True, subtract_mean=True)
return X, y, nb_samples, num_categories
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--extract_data', dest='extract_data', action='store_const', const=True, default=False)
parser.add_argument('--run', dest='run', action='store_const', const=True, default=False)
parser.add_argument('--continue', dest='continue_training', action='store_const', const=True, default=False)
args = parser.parse_args()
extract_mode = args.extract_data
run_mode = args.run
continue_ = args.continue_training
if run_mode:
run(extract=extract_mode, cont=continue_)
elif extract_mode:
extract_data()