深度神经网络

from sklearn import datasets
import numpy as np
import matplotlib.pyplot as plt

# 一、数据集导入和数据预处理
iris_data = datasets.load_iris()
input_data = iris_data.data
correct = iris_data.target

n_data = len(correct)

# 标准化处理
ave_input = np.average(input_data,axis = 0) #是矩阵
std_input = np.std(input_data,axis = 0) #是向量
input_data = (input_data-ave_input)/std_input

# 将数据转化为独热编码格式
correct_data = np.zeros((n_data,3))
for i in range(n_data):
    correct_data[i,correct[i]] = 1.0

# 分割训练集和测试集
index = np.arange(n_data)
index_train = index[index%2==0]
index_test = index[index%2 !=0]

input_train = input_data[index_train,:]
correct_train = correct_data[index_train,:]
input_test = input_data[index_test,:]
correct_test = correct_data[index_test,:]

n_train = input_train.shape[0]
n_test = input_test.shape[0]

#基础值的设置
n_in = 4
n_mid = 25
n_out = 3

wb_width = 0.1 #权重和偏置的分散度
eta = 0.01
epoch = 1000 #为了时间和性能考虑，测试时可以只设置100，但是自己跑毕设实验得原数据！！！
batch_size = 8
interval = 100

# 二、各个网络层的实现，抽象出基类，感觉更多的东西可以被抽象？？？
#神经层的基类
class BaseLayer:
    def __init__(self,n_upper,n):
        self.w = wb_width * np.random.randn(n_upper,n)
        self.b = wb_width * np.random.randn(n)
        
    def update(self,eta):
        self.w -= eta*self.grad_w
        self.b -= eta*self.grad_b 
        
class MiddleLayer(BaseLayer):
    def forward(self,x):
        self.x = x
        self.u = np.dot(x,self.w) + self.b   #这里的u加上了self是因为有多个中间层！输出层只有一个所以不用self来约束
        self.y = np.where(self.u <=0,0,self.u)   #使用ReLU
    
    def backward(self,grad_y):
        delta = grad_y * np.where(self.u<=0,0,1)  #ReLU 的微分
        self.grad_w = np.dot(self.x.T,delta)
        self.grad_b = np.sum(delta,axis=0)
        self.grad_x = np.dot(delta,self.w.T)
        
class OutputLayer(BaseLayer):        
    def forward(self,x):
        self.x = x
        u = np.dot(x,self.w) + self.b
        self.y = np.exp(u)/np.sum(np.exp(u),axis=1,keepdims=True)   #使用SoftMax
    
    def backward(self,t):
        delta = self.y -t  #SoftMax 的微分
        self.grad_w = np.dot(self.x.T,delta)
        self.grad_b = np.sum(delta,axis=0)
        self.grad_x = np.dot(delta,self.w.T)

#三、神经网络的实现
middle_layer_1 = MiddleLayer(n_in,n_mid)
middle_layer_2 = MiddleLayer(n_mid,n_mid)
output_layer = OutputLayer(n_mid,n_out)

#由于有多层 所以对于正向处理和反向处理这些都封装一下
def forward_propagation(x):
    middle_layer_1.forward(x)
    middle_layer_2.forward(middle_layer_1.y)
    output_layer.forward(middle_layer_2.y)
    
def backward_propagation(t):
    output_layer.backward(t)
    middle_layer_2.backward(output_layer.grad_x)
    middle_layer_1.backward(middle_layer_2.grad_x)
    
def update_wb():
    middle_layer_1.update(eta)
    middle_layer_2.update(eta)
    output_layer.update(eta)
    
#计算交叉熵误差
def get_error(t,batch_size):
    return -np.sum(t*np.log(output_layer.y+1e-7))/batch_size

#记录误差
train_error_x = []
train_error_y = []
test_error_x = []
test_error_y = []

#记录学习的过程和经过
n_batch = n_train // batch_size  #取整数部分参与计算，每一轮epoch的批次尺寸
for i in range(epoch):
    #误差的统计和测算
    forward_propagation(input_train)
    error_train = get_error(correct_train,n_train)
    forward_propagation(input_test)
    error_test = get_error(correct_test,n_test)
    #误差的记录
    test_error_x.append(i)
    test_error_y.append(error_test)
    train_error_x.append(i)
    train_error_y.append(error_train)
    #进度的显示
    if i%interval==0:
        print("Epoch:"+str(i)+"/"+str(epoch),"Error_train:"+str(error_train),"Error_test:"+str(error_test))
    
    #学习
    index_random = np.arange(n_train)  #打乱索引值 这里也是基于随机梯度下降法
    np.random.shuffle(index_random)
    for j in range(n_batch):
        #取出最小批次
        mb_index= index_random[j*batch_size:(j+1)*batch_size]
        x = input_train[mb_index,:]
        t = correct_train[mb_index,:]
        
        forward_propagation(x)
        backward_propagation(t)
        
        update_wb()
        
    plt.plot(train_error_x,train_error_y,label="Train")
    plt.plot(test_error_x,test_error_y,label="Test")
    plt.legend() #设置图例

    plt.xlabel("Epoches")
    plt.ylabel("Error")

    plt.show()

# 计算正确率
forward_propagation(input_train)
count_train = np.sum(np.argmax(output_layer.y,axis=1) == np.argmax(correct_train,axis=1))

forward_propagation(input_test)
count_test = np.sum(np.argmax(output_layer.y,axis=1) == np.argmax(correct_test,axis=1))
print("Accuracy Train:"+str(count_train/n_train*100)+"%","Accuracy Test:"+str(count_test/n_test*100)+"%")

class BaseLayer:
    def __init__(self,n_upper,n):
        self.w = wb_width * np.random.randn(n_upper,n)
        self.b = wb_width * np.random.randn(n)
        
        self.h_w = np.zeros((n_upper,n))+1e-8
        self.h_b = np.zeros(n)+1e-8
        
    def update(self,eta):
        self.h_w += self.grad_w*self.grad_w
        self.h_b += self.grad_b*self.grad_b
        
        self.w -= eta/np.sqrt(self.h_w)*self.grad_w
        self.b -= eta/np.sqrt(self.h_b)*self.grad_b
        

#封装一个Dropout层 放在每一个中间层后面
class Dropout:
    def __init__(self,dropout_ratio):
         self.dropout_ratio = dropout_ratio
    
    def forward(self,x,is_train):
        if is_train: 
            #学习时，创建与输入具有相同形状的随机数矩阵
            rand = np.random.rand(*x.shape)
            #将对应的神经元设置为有效1，无效0，保存在self.dropout矩阵中
            self.dropout = np.where(rand>self.dropout_ratio,1,0) 
            self.y = x*self.dropout #将上层网络中的神经元随机设置成无效的
        else:
            self.y = (1-self.dropout_ratio)*x #使输出下降
            
    def backward(self,grad_y):
        self.grad_x = grad_y*self.dropout #确保不会对无效的神经元进行反向传播处理
        
   ##中间层加入时这样做：
   dropout_1 = Dropout(0.5)   
   
   ##正向传播中
   dropout_1.forward(middle_layer_1,is_train) 
   middle_layer_2.forward(dropout_1.y) 
   
   ##反向传播中
   dropout_2.backward(output_layer.grad_x) 
   middle_layer_2.backward(dropout_2.grad_x)