写在前面

前几天看了篇NLP的论文，写了篇论文阅读笔记，所以耽搁了几天，今天来继续更新啦~
今天主要是分享将纯python代码文件在GPU上跑起来，从而实现加速的程序，记录一下numba库的使用方法，在此方法基础上实现之前分享的神经网络。

昨天遗留

1：我将MNIST的训练集文件取出了3.2w张（6w张训练时间实在是太久了），一个epoch训练3.2w张图片，并且在一个epoch中训练集换成了320份，每份100张，每次训练的时候会记录这100张训练集上的参数求均值再更新，这样可以使得梯度下降得更高效；

2：学习率对于模型表现的影响将在下面进行分析；

今日工作

今天主要探索了学习率对于模型结果的影响，先来看下面几张图片：




在保证其他参数不变的情况下，我分别将学习率设置成了0.2，0.25，0.3，0.4，分别对应从上到下的结果图，从图中可以很明显的看出来随着学习率的上升，模型收敛的速度越快，根据在小测试集上的不同表现，我选择了学习率0.5。

还有关于numba文件库的使用，jit只能加速numpy的函数和循环操作，因此我将我的主要参数更新函数单独抽象出来了。

@jit
def TrainBody(w, v, gamma, theta_1, theta_2, theta_3, x,\
            dw, dv, dgamma, dtheta_1, dtheta_2, dtheta_3, \
                m, m_out, n, n_out, y, y_out, y_pre):

    for i in range(total_y):
        dtheta_3[0][i] = dtheta_3[0][i] + (y[0][i] - y_out[0][i]) * dsigmoid(y_pre[0][i])
    for i in range(n_num):
        for k in range(total_y):
            dtheta_2[0][i] = dtheta_2[0][i] + (y[0][k] - y_out[0][k]) * dsigmoid(y_pre[0][k]) * \
                        gamma[i][k] * dReLU(n[0][i])
    for i in range(m_num):
        for j in range(total_y):
            for k in range(n_num):
                dtheta_1[0][i] = dtheta_1[0][i] + (y[0][j] - y_out[0][j]) * dsigmoid(y_pre[0][j]) * \
                            gamma[k][j] * dReLU(n[0][k]) * v[i][k] * dReLU(m[0][i])
    for i in range(n_num):
        for j in range(total_y):
            dgamma[i][j] = dgamma[i][j] + (y_out[0][j] - y[0][j]) * n_out[0][i] * dsigmoid(y_pre[0][j])
    for i in range(m_num):
        for j in range(n_num):
            for k in range(total_y):
                dv[i][j] = dv[i][j] + (y_out[0][k] - y[0][k]) * gamma[j][k] * dsigmoid(y_pre[0][k]) * \
                    m_out[0][i] * dReLU(n[0][j])
    for i in range(total_x):
        for j in range(m_num):
            for k in range(total_y):
                for l in range(n_num):
                    dw[i][j] = dw[i][j] + (y_out[0][k] - y[0][k]) * dsigmoid(y_pre[0][k]) * gamma[l][k] * \
                            dReLU(n[0][l]) * v[j][l] * dReLU(m[0][j]) * x[i]

    return dw, dv, dgamma, theta_1, theta_2, theta_3

并且我还写了测试模型数据的函数，实现如下：

def VerifyModel(verify_img, verify_lab):
    right_cnt = 0
    all_img = np.zeros(((verify_n, total_x)), dtype=float)

    for i in range(verify_n):
        all_img[i, :] = normalization((verify_img[i].flatten()).astype(float))

    w = np.loadtxt("w.csv", delimiter=",")
    v = np.loadtxt("v.csv", delimiter=",")
    gamma= np.loadtxt("gamma.csv", delimiter=",")
    theta_1 = np.loadtxt("theta_1.csv", delimiter=",")
    theta_2 = np.loadtxt("theta_2.csv", delimiter=",")
    theta_3 = np.loadtxt("theta_3.csv", delimiter=",")

    for i in range(verify_n):
        x = all_img[i]
        max_y = 0
        out_put_num = 10
        m = np.dot(x, w) - theta_1
        for j in range(m_num):
            m_out[0][j] = ReLU(m[j])
        n = np.dot(m_out, v) - theta_2
        for j in range(n_num):
            n_out[0][j] = ReLU(n[0][j])
        y_pre = np.dot(n, gamma) - theta_3
        for j in range(total_y):
            y_out[0][j] = sigmoid(y_pre[0][j])
            if y_out[0][j] > 0.8:
                out_put_num = j
                break
        if out_put_num == verify_lab[i]:
            right_cnt = right_cnt + 1

    print("right/total: %d/%d ,rightpercent: %.3f" % (right_cnt , verify_n, (right_cnt/verify_n)))

明天问题

1：初步使用ROC曲线等方法评估模型的性能；

总体程序：

# from inputDataSet import DataSetProcess
from numba import jit
import numpy as np
import struct
# import matplotlib.pyplot as plt
import random
import datetime
import time
# import math
# import tensorflow as tf

total_n = 60000
verify_n = 10000
train_aside_n = 32000
epoch = 20
division = 100
study_step = 0.5
sub_train = train_aside_n / division
start_rand_max = 0.1
img_size_bit = struct.calcsize('>784B')
lab_size_bit = struct.calcsize('>1B')
n_num = 16
m_num = 16
total_x = 784
total_y = 10

# @jit
def readFile(type=0):  # 0 is traindata,1 is testdata
    if (type == 0):
        with open('./dataSet/train-images.idx3-ubyte', 'rb') as ti:
            train_image = ti.read()
        with open('./dataSet/train-labels.idx1-ubyte', 'rb') as tl:
            train_labels = tl.read()

        return train_image, train_labels
    elif (type == 1):
        with open('./dataSet/t10k-images.idx3-ubyte', 'rb') as t_i:
            test_image = t_i.read()
        with open('./dataSet/t10k-labels.idx1-ubyte', 'rb') as t_l:
            test_labels = t_l.read()
        return test_image, test_labels

# @jit
def getImages(image, n, startidx=0):
    img = []
    index = struct.calcsize('>IIII') + img_size_bit * startidx
    for i in range(n):
        temp = struct.unpack_from('>784B', image, index)
        img.append(np.reshape(temp, (28, 28)))
        index += img_size_bit
    return img

# @jit
def getLabels(label, n, startidx=0):
    lab = []
    index = struct.calcsize('>II') + lab_size_bit * startidx
    for i in range(n):
        temp = struct.unpack_from('>1B', label, index)
        lab.append(temp[0])
        index += lab_size_bit
    return lab

@jit
def sigmoid(x):
    s = 1 / (1 + np.exp(-x))
    # print("sig:", s)
    return s

@jit
def dsigmoid(x):
    s = sigmoid(x) * (1 - sigmoid(x))
    return s

@jit
def ReLU(x):
    if x > 0:
        s = x
    else:
        s = 0
    return s

@jit
def dReLU(x):
    if x > 0:
        s = 1
    else:
        s = 0
    return s

@jit
def  normalization(x):
    max = float(0)
    min = float(999)
    for i in range(0, total_x):
        if (x[i] > max):
            max = x[i]
        elif (x[i] < min):
            min = x[i]

    for i in range(0, total_x):
        x[i] = (x[i] - min) / (max - min)
    # print(x)
    return x

# @jit
def VerifyModel(verify_img, verify_lab):
#     print("VerifyModel start")
    right_cnt = 0
    all_img = np.zeros(((verify_n, total_x)), dtype=float)
#     print(verify_lab)

    for i in range(verify_n):
        all_img[i, :] = normalization((verify_img[i].flatten()).astype(float))

    w = np.loadtxt("w.csv", delimiter=",")
    v = np.loadtxt("v.csv", delimiter=",")
    gamma= np.loadtxt("gamma.csv", delimiter=",")
    theta_1 = np.loadtxt("theta_1.csv", delimiter=",")
    theta_2 = np.loadtxt("theta_2.csv", delimiter=",")
    theta_3 = np.loadtxt("theta_3.csv", delimiter=",")

    for i in range(verify_n):
        x = all_img[i]
        max_y = 0
        out_put_num = 10
        m = np.dot(x, w) - theta_1
        for j in range(m_num):
            m_out[0][j] = ReLU(m[j])
        n = np.dot(m_out, v) - theta_2
        for j in range(n_num):
            n_out[0][j] = ReLU(n[0][j])
        y_pre = np.dot(n, gamma) - theta_3
        for j in range(total_y):
            y_out[0][j] = sigmoid(y_pre[0][j])
            if y_out[0][j] > 0.8:
                out_put_num = j
                break
        if out_put_num == verify_lab[i]:
            right_cnt = right_cnt + 1

    print("right/total: %d/%d ,rightpercent: %.3f" % (right_cnt , verify_n, (right_cnt/verify_n)))

@jit
def TrainBody(w, v, gamma, theta_1, theta_2, theta_3, x,\
            dw, dv, dgamma, dtheta_1, dtheta_2, dtheta_3, \
                m, m_out, n, n_out, y, y_out, y_pre):

    for i in range(total_y):
        dtheta_3[0][i] = dtheta_3[0][i] + (y[0][i] - y_out[0][i]) * dsigmoid(y_pre[0][i])
    for i in range(n_num):
        for k in range(total_y):
            dtheta_2[0][i] = dtheta_2[0][i] + (y[0][k] - y_out[0][k]) * dsigmoid(y_pre[0][k]) * \
                        gamma[i][k] * dReLU(n[0][i])
    for i in range(m_num):
        for j in range(total_y):
            for k in range(n_num):
                dtheta_1[0][i] = dtheta_1[0][i] + (y[0][j] - y_out[0][j]) * dsigmoid(y_pre[0][j]) * \
                            gamma[k][j] * dReLU(n[0][k]) * v[i][k] * dReLU(m[0][i])
    for i in range(n_num):
        for j in range(total_y):
            dgamma[i][j] = dgamma[i][j] + (y_out[0][j] - y[0][j]) * n_out[0][i] * dsigmoid(y_pre[0][j])
    for i in range(m_num):
        for j in range(n_num):
            for k in range(total_y):
                dv[i][j] = dv[i][j] + (y_out[0][k] - y[0][k]) * gamma[j][k] * dsigmoid(y_pre[0][k]) * \
                    m_out[0][i] * dReLU(n[0][j])
    for i in range(total_x):
        for j in range(m_num):
            for k in range(total_y):
                for l in range(n_num):
                    dw[i][j] = dw[i][j] + (y_out[0][k] - y[0][k]) * dsigmoid(y_pre[0][k]) * gamma[l][k] * \
                            dReLU(n[0][l]) * v[j][l] * dReLU(m[0][j]) * x[i]

    return dw, dv, dgamma, theta_1, theta_2, theta_3


    # print(type(theta_1))

if __name__ == '__main__':
    print("BPnet")
    time1 = time.time()
    start_train_idx = 5000
    # take os time as random seed
    random.seed(datetime.datetime.now())

    image, label = readFile()
    train_img = getImages(image, train_aside_n, start_train_idx)
    train_lab = getLabels(label, train_aside_n, start_train_idx)  # set aside deal

    image_verify, label_verify = readFile(1)
    verify_img = getImages(image_verify, verify_n)
    verify_lab = getLabels(label_verify, verify_n)  # set verify deal

    all_img = np.zeros(((train_aside_n, total_x)), dtype=float)

    for i in range(train_aside_n):
        all_img[i, :] = normalization((train_img[i].flatten()).astype(float))

    # we can change train data numbers here
    train_n = train_aside_n

    # showImgLabel(train_aside_n,train_img,train_lab)
    #we get the data but can not use directly,we get tuple,but we use array
    w = np.loadtxt("w.csv", delimiter=",")
    v = np.loadtxt("v.csv", delimiter=",")
    gamma = np.loadtxt("gamma.csv", delimiter=",")
    theta_1 = np.loadtxt("theta_1.csv", delimiter=",")
    theta_2 = np.loadtxt("theta_2.csv", delimiter=",")
    theta_3 = np.loadtxt("theta_3.csv", delimiter=",")

    dw = np.zeros(((total_x, m_num)), dtype=float)  # (()) is used to confirm line and row
    dv = np.zeros((m_num, n_num), dtype=float)
    dgamma = np.zeros((n_num, total_y), dtype=float)
    dtheta_1 = np.zeros(((1, m_num)), dtype=float)
    dtheta_2 = np.zeros(((1, n_num)), dtype=float)
    dtheta_3 = np.zeros(((1, total_y)), dtype=float)

    #gradient
    m = np.ones(((1, m_num)), dtype=float)
    m_out = np.ones(((1, m_num)), dtype=float)
    n = np.ones(((1, n_num)), dtype=float)
    n_out = np.ones(((1, n_num)), dtype=float)
    y = np.ones((1, total_y), dtype=float)  # read mark
    y_out = np.ones((1, total_y), dtype=float)
    y_pre = np.ones((1, total_y), dtype=float)

    #make the data dimension same
    w = w - (study_step * dw) / division
    v = v - (study_step * dv) / division
    gamma = gamma - (study_step * dgamma) / division
    theta_1 = theta_1 - (study_step * dtheta_1) / division
    theta_2 = theta_2 - (study_step * dtheta_2) / division
    theta_3 = theta_3 - (study_step * dtheta_3) / division

    det_Ek_v = np.zeros(epoch, dtype=float)
    Ek_cnt = 0

    for i in range(0, epoch):
        det_Ek_v[Ek_cnt] = np.mean(np.square(y_out - y)) * 0.5
        Ek_cnt = Ek_cnt + 1
        print("epoch: %d,cost: %.2fs" % (i, time.time() - time1))
        VerifyModel(verify_img, verify_lab)
        time1 = time.time()
        for k in range(0, train_n):
            x = all_img[k]
            y = np.zeros((1, total_y), dtype=float)  # read mark
            y[0][train_lab[k]] = 1

            # update values
            m = np.dot(x, w) - theta_1
            for i in range(0, m_num):
                m_out[0][i] = ReLU(m[0][i])
            n = np.dot(m_out, v) - theta_2
            for i in range(0, n_num):
                n_out[0][i] = ReLU(n[0][i])
            y_pre = np.dot(n, gamma) - theta_3
            for i in range(total_y):
                y_out[0][i] = sigmoid(y_pre[0][i])

            (dw, dv, dgamma, theta_1, theta_2, theta_3) = TrainBody(w, v, gamma, theta_1, theta_2, theta_3, x,\
                dw, dv, dgamma, dtheta_1, dtheta_2, dtheta_3, \
                    m, m_out, n, n_out, y, y_out, y_pre)

        # update dw,dv,dgama,dtheta_1,dtheta_2,dtheta_3
            if (k + 1) % division == 0:
                # update w,v,gama,theta_1,theta_2,theta_3
                w = w - (study_step * dw) / division
                v = v - (study_step * dv) / division
                gamma = gamma - (study_step * dgamma) / division
                theta_1 = theta_1 - (study_step * dtheta_1) / division
                theta_2 = theta_2 - (study_step * dtheta_2) / division
                theta_3 = theta_3 - (study_step * dtheta_3) / division

                dw = np.zeros(((total_x, m_num)), dtype=float)  # (()) is used to confirm line and row
                dv = np.zeros((m_num, n_num), dtype=float)
                dgamma = np.zeros((n_num, total_y), dtype=float)
                dtheta_1 = np.zeros(((1, m_num)), dtype=float)
                dtheta_2 = np.zeros(((1, n_num)), dtype=float)
                dtheta_3 = np.zeros(((1, total_y)), dtype=float)
#         save parameters
        np.savetxt("w.csv", w, delimiter=",")
        np.savetxt("v.csv", v, delimiter=",")
        np.savetxt("gamma.csv", gamma, delimiter=",")
        np.savetxt("theta_1.csv", theta_1, delimiter=",")
        np.savetxt("theta_2.csv", theta_2, delimiter=",")
        np.savetxt("theta_3.csv", theta_3, delimiter=",")
    np.savetxt("det_Ek_v.csv", det_Ek_v, delimiter=",")