# -*- coding:utf-8 -*- # 用于修剪网络模型 import numpy as np import matplotlib.pyplot as plt import os os.environ['GLOG_minloglevel'] = '2' import caffe # 由稠密变成CSC稀疏矩阵 def dense_to_sparse_csc(W_flatten, num_level): # W_flatten: 扁平化的权重矩阵 # num_level: 量化级别 csc_W = [] # 存储稀疏矩阵 csc_indx = [] indx = 0 for n in range(len(W_flatten)): if W_flatten[n]!=0 or indx == 2**num_level: csc_W.append(W_flatten[n]) csc_indx.append(indx) indx = 0 else: indx += 1 if indx!=0: csc_W.append(0.0) csc_indx.append(indx-1) return np.array(csc_W, dtype=np.float32),np.array(csc_indx, dtype=np.int8) # 由稠密变成CSC稀疏矩阵 def sparse_to_dense_csc(csc_W, csc_W_indx): # W_flatten: 扁平化的权重矩阵 # num_level: 量化级别 W_flatten = [] # 存储稠密矩阵 indx = 0 for n in range(len(csc_W)): if csc_W_indx[n]!=0: W_flatten.extend([0]*(csc_W_indx[n])) W_flatten.append(csc_W[n]) return np.array(W_flatten, dtype=np.float32) def read_sparse_net(filename, net, layers): pass def write_sparse_net(filename, net): pass # 画出各层参数的直方图 def draw_hist_weight(net, layers): plt.figure() # 画图 layer_num = len(layers) for i, layer in enumerate(layers): i += 1 W = net.params[layer][0].data plt.subplot(layer_num/2, 2, i) numBins = 2 ^ 5 plt.hist(W.flatten(), numBins, color='blue', alpha=0.8) plt.title(layer) plt.show() # 网络模型的参数 def analyze_param(net, layers): print '\n=============analyze_param start===============' total_nonzero = 0 total_allparam = 0 percentage_list = [] for i, layer in enumerate(layers): i += 1 W = net.params[layer][0].data b = net.params[layer][1].data print 'W(%s) range = [%f, %f]' % (layer, min(W.flatten()), max(W.flatten())) print 'W(%s) mean = %f, std = %f' % (layer, np.mean(W.flatten()), np.std(W.flatten())) non_zero = (np.count_nonzero(W.flatten()) + np.count_nonzero(b.flatten())) # 参数非零值 all_param = (np.prod(W.shape) + np.prod(b.shape)) # 所有参数的数目 this_layer_percentage = non_zero / float(all_param) # 参数比例 total_nonzero += non_zero total_allparam += all_param print 'non-zero W and b cnt = %d' % non_zero print 'total W and b cnt = %d' % all_param print 'percentage = %f\n' % (this_layer_percentage) percentage_list.append(this_layer_percentage) print '=====> summary:' print 'non-zero W and b cnt = %d' % total_nonzero print 'total W and b cnt = %d' % total_allparam print 'percentage = %f' % (total_nonzero / float(total_allparam)) print '=============analyze_param ends ===============' return (total_nonzero / float(total_allparam), percentage_list) def prune(threshold, test_net, layers): sqarse_net = {} for i, layer in enumerate(layers): print '\n============ Pruning %s : threshold=%0.2f ============' % (layer,threshold[i]) W = test_net.params[layer][0].data b = test_net.params[layer][1].data hi = np.max(np.abs(W.flatten())) hi = np.sort(-np.abs(W.flatten()))[int((len(W.flatten())-1)* threshold[i])] # abs(val) = 0 ==> 0 # abs(val) >= threshold ==> 1 interpolated = np.interp(np.abs(W), [0, hi * threshold[i], 999999999.0], [0.0, 1.0, 1.0]) # 小于阈值的权重被随机修剪 random_samps = np.random.rand(len(W.flatten())) random_samps.shape = W.shape # 修剪阈值 # mask = (random_samps < interpolated) mask = (np.abs(W) > (np.abs(hi))) mask = np.bool_(mask) W = W * mask print 'non-zero W percentage = %0.5f ' % (np.count_nonzero(W.flatten()) / float(np.prod(W.shape))) # 保存修剪后的阈值 test_net.params[layer][0].data[...] = W # net.params[layer][0].mask[...] = mask csc_W, csc_W_indx = dense_to_sparse_csc(W.flatten(), 8) dense_W = sparse_to_dense_csc(csc_W, csc_W_indx) sqarse_net[layer + '_W'] = csc_W sqarse_net[layer + '_W_indx'] = csc_W_indx # 计算修剪后的权重稀疏度 # np.savez(model_dir + model_name +"_crc.npz",sqarse_net) # 保存存储成CRC格式的稀疏网络 (total_percentage, percentage_list) = analyze_param(test_net, layers) test_loss, accuracy = test_net_accuracy(test_net) return (threshold, total_percentage, percentage_list, test_loss, accuracy) def test_net_accuracy(test_net): test_iter = 100 test_loss = 0 accuracy = 0 for test_it in range(test_iter): # 进行一次测试 test_net.forward() # 计算test loss test_loss += test_net.blobs['loss'].data # 计算test accuracy accuracy += test_net.blobs['accuracy'].data return (test_loss / test_iter), (accuracy / test_iter) def eval_prune_threshold(threshold_list, test_prototxt, caffemodel, prune_layers): def net_prune(threshold, test_prototx, caffemodel, prune_layers): test_net = caffe.Net(test_prototx, caffemodel, caffe.TEST) return prune(threshold, test_net, prune_layers) accuracy = [] for threshold in threshold_list: results = net_prune(threshold, test_prototxt, caffemodel, prune_layers) print 'threshold: ', results[0] print '\ntotal_percentage: ', results[1] print '\npercentage_list: ', results[2] print '\ntest_loss: ', results[3] print '\naccuracy: ', results[4] accuracy.append(results[4]) plt.plot(accuracy,'r.') plt.show() # 迭代训练修剪后网络 def retrain_pruned(solver, pruned_caffemodel, threshold, prune_layers): #solver = caffe.SGDSolver(solver_proto) retrain_iter = 20 accuracys = [] for i in range(retrain_iter): solver.net.copy_from(pruned_caffemodel) # solver.solve() solver.step(500) _,_,_,_,accuracy=prune(threshold, solver.test_nets[0], prune_layers) solver.test_nets[0].save(pruned_caffemodel) accuracys.append(accuracy) plt.plot(accuracys, 'r.-') plt.show() #CPU或GPU模型转换 #caffe.set_mode_cpu() caffe.set_device(0) caffe.set_mode_gpu() caffe_root = '../../' #model_dir = caffe_root + 'models/SSD_300x300/' #deploy = model_dir + 'deploy.prototxt' #model_name = 'VGG_VOC0712_SSD_300x300_iter_60000' #caffemodel = model_dir + model_name + '.caffemodel' model_dir = caffe_root + 'models/mnist/' deploy = model_dir + 'deploy.prototxt' model_name = 'LeNet5_Mnist_shapshot_iter_10000' caffemodel = model_dir + model_name + '.caffemodel' test_prototxt = model_dir + 'test.prototxt' solver_proto = model_dir + 'solver.prototxt' solver = caffe.SGDSolver(solver_proto) # 要修剪的层 prune_layers = ['conv1','conv2','ip1','ip2'] # 测试修剪率 test_threshold_list = [[0.3, 1 ,1 ,1], [0.4, 1 ,1 ,1], [0.5, 1 ,1 ,1], [0.6, 1 ,1 ,1], [0.7, 1 ,1 ,1], [1, 0.05, 1, 1], [1, 0.1, 1, 1], [1, 0.15, 1, 1], [1, 0.2, 1, 1], [1, 0.3, 1, 1], [1, 1, 0.05, 1], [1, 1, 0.1, 1], [1, 1, 0.15, 1], [1, 1, 0.2, 1], [1, 1, 0.3, 1], [1, 1, 1, 0.05], [1, 1, 1, 0.1], [1, 1, 1, 0.15], [1, 1, 1, 0.2], [1, 1, 1, 0.3]] # 验证修剪率 #eval_prune_threshold(test_threshold_list, test_prototxt, caffemodel, prune_layers) threshold = [0.3, 0.1, 0.01, 0.2] prune(threshold, solver.test_nets[0], prune_layers) pruned_model = model_dir + model_name +'_pruned' + '.caffemodel' solver.test_nets[0].save(pruned_model) retrain_pruned(solver, pruned_model, threshold, prune_layers) """ # 各层对应的修剪率 threshold = [0.3, 0.1, 0.01, 0.2] net = caffe.Net(deploy, caffemodel, caffe.TEST) # 修剪 prune(threshold, net, prune_layers, test_prototxt) # 保存修剪后的稀疏网络模型 output_model = model_name +'_pruned' + '.caffemodel' net.save(output_model) """