Repository navigation
Expand file tree
/
Copy pathramp_ups.py
More file actions
75 lines (61 loc) · 2.03 KB
/
Copy pathramp_ups.py
File metadata and controls
75 lines (61 loc) · 2.03 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
import numpy as np
def pseudo_rampup(T1, T2):
def warpper(epoch):
if epoch > T1:
alpha = (epoch-T1) / (T2-T1)
if epoch > T2:
alpha = 1.0
else:
alpha = 0.0
return alpha
return warpper
def exp_rampup(rampup_length):
"""Exponential rampup from https://arxiv.org/abs/1610.02242"""
def warpper(epoch):
if epoch < rampup_length:
epoch = np.clip(epoch, 0.0, rampup_length)
phase = 1.0 - epoch / rampup_length
return float(np.exp(-5.0 * phase * phase))
else:
return 1.0
return warpper
def linear_rampup(rampup_length):
"""Linear rampup"""
def warpper(epoch):
if epoch < rampup_length:
return epoch / rampup_length
else:
return 1.0
return warpper
def exp_rampdown(rampdown_length, num_epochs):
"""Exponential rampdown from https://arxiv.org/abs/1610.02242"""
def warpper(epoch):
if epoch >= (num_epochs - rampdown_length):
ep = .5* (epoch - (num_epochs - rampdown_length))
return float(np.exp(-(ep * ep) / rampdown_length))
else:
return 1.0
return warpper
def cosine_rampdown(rampdown_length, num_epochs):
"""Cosine rampdown from https://arxiv.org/abs/1608.03983"""
def warpper(epoch):
if epoch >= (num_epochs - rampdown_length):
ep = .5* (epoch - (num_epochs - rampdown_length))
return float(.5 * (np.cos(np.pi * ep / rampdown_length) + 1))
else:
return 1.0
return warpper
def exp_warmup(rampup_length, rampdown_length, num_epochs):
rampup = exp_rampup(rampup_length)
rampdown = exp_rampdown(rampdown_length, num_epochs)
def warpper(epoch):
return rampup(epoch)*rampdown(epoch)
return warpper
# def test_warmup():
if __name__ == '__main__':
# warmup = exp_warmup(80, 50, 100)
warmup = exp_rampup(11)
warmup = linear_rampup(11)
for ep in range(50):
print(ep, warmup(ep))
# test_warmup()