neural networks

神经网络的原理与使用

激活函数 - 非线性矫正

import numpy as np
import matplotlib.pyplot as plt 

# 生成一个等差数列
x=np.linspace(-5, 5, 200)

# 画出非线性矫正图形
plt.plot(x, np.tanh(x), label="tanh")
plt.plot(x, np.maximum(x, 0), label="relu")

plt.legend(loc="best")
#plt.xlabel("x")
#plt.ylabel("Y")
plt.show()

神经网络的参数设置

help(MLPClassifier)

class MLPClassifier(sklearn.base.ClassifierMixin, BaseMultilayerPerceptron) | MLPClassifier(hidden_layer_sizes=(100,), activation='relu', *, solver='adam', alpha=0.0001, batch_size='auto', learning_rate='constant', learning_rate_init=0.001, power_t=0.5, max_iter=200, shuffle=True, random_state=None, tol=0.0001, verbose=False, warm_start=False, momentum=0.9, nesterovs_momentum=True, early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08, n_iter_no_change=10, max_fun=15000)

# 导入多层感知机 MPL 神经网络
from sklearn.neural_network import MLPClassifier
# 导入wine数据集
from sklearn.datasets import load_wine
wine=load_wine()
X=wine.data[:, :2]
y=wine.target

# 拆分数据
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

# 拟合
mlp=MLPClassifier(solver="lbfgs", max_iter=300, random_state=7)
mlp.fit(X_train, y_train)

# 打分
print("training set:{:0.3f}".format( mlp.score(X_train, y_train)) )
print("tesing set:{:0.3f}".format( mlp.score(X_test, y_test)) )

training set:0.827
tesing set:0.822

# 可视化
import matplotlib.pyplot as plt 
from matplotlib.colors import ListedColormap

# 使用不同色块表示不同分类
cmap_light=ListedColormap(["#FFAAAA", "#AAFFAA", "#AAAAFF"])
cmap_bold=ListedColormap(["FF0000", "00FF00", "0000FF"])
x_min, x_max = X_train[:, 0].min()-1, X_train[:, 0].max()+1
y_min, y_max = X_train[:, 1].min()-1, X_train[:, 1].max()+1
xx, yy=np.meshgrid(np.arange(x_min, x_max, 0.02),
                  np.arange(y_min, y_max, 0.02))
Z=mlp.predict(np.c_[xx.ravel(), yy.ravel()])
Z=Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cmap_light, shading='auto')

# 散点图
plt.scatter(X[:, 0], X[:, 1], c=y, edgecolor="k", s=60)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("MLPClassifier:solver=lbfgs")
plt.show()

增加模型复杂度的4种方法

增加隐藏层node数

# 我们验证另一面：减少node能使模型简化。


# 拟合
mlp_20=MLPClassifier(solver="lbfgs", hidden_layer_sizes =[10], max_iter=800, random_state=8)
mlp_20.fit(X_train, y_train)

# 打分
print("training set:{:0.3f}".format( mlp_20.score(X_train, y_train)) )
print("tesing set:{:0.3f}".format( mlp_20.score(X_test, y_test)) )


# 画图
Z=mlp_20.predict(np.c_[xx.ravel(), yy.ravel()])
Z=Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cmap_light, shading='auto')

# 散点图
plt.scatter(X[:, 0], X[:, 1], c=y, edgecolor="k", s=60)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("MLPClassifier:nodes=10")
plt.show()

training set:0.782
tesing set:0.822

• 决定边界丢失了很多细节。每一个隐藏层中，node数大概决定边界的最大直线数，这个值越大，边界越平滑。
• 还有2个方法可以让边界平滑
  • 增加隐藏层数量
  • 激活函数改为 tanh

隐藏层增加到2层

# 拟合
mlp_2L=MLPClassifier(solver="lbfgs", hidden_layer_sizes =[10, 10], max_iter=800, random_state=8)
mlp_2L.fit(X_train, y_train)

# 打分
print("training set:{:0.3f}".format( mlp_2L.score(X_train, y_train)) )
print("tesing set:{:0.3f}".format( mlp_2L.score(X_test, y_test)) )


# 画图
Z=mlp_2L.predict(np.c_[xx.ravel(), yy.ravel()])
Z=Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cmap_light, shading='auto')

# 散点图
plt.scatter(X[:, 0], X[:, 1], c=y, edgecolor="k", s=60)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("MLPClassifier:nodes=[10, 10]")
plt.show()

# 边界有更多细节了

training set:0.835
tesing set:0.822

激活函数 activation='tanh'

# 拟合
mlp_2L=MLPClassifier(solver="lbfgs", hidden_layer_sizes =[10, 10], activation='tanh',
                     max_iter=200, random_state=7)
mlp_2L.fit(X_train, y_train)

# 打分
print("training set:{:0.3f}".format( mlp_2L.score(X_train, y_train)) )
print("tesing set:{:0.3f}".format( mlp_2L.score(X_test, y_test)) )


# 画图
Z=mlp_2L.predict(np.c_[xx.ravel(), yy.ravel()])
Z=Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cmap_light, shading='auto')

# 散点图
plt.scatter(X[:, 0], X[:, 1], c=y, edgecolor="k", s=60)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("MLPClassifier:nodes=[10, 10] activation='tanh'")
plt.show()

# 边界有更多细节了

/home/wangjl/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:549: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
  self.n_iter_ = _check_optimize_result("lbfgs", opt_res, self.max_iter)

training set:0.782
tesing set:0.844

提高 alpha 值(默认 0.0001)

# 拟合
mlp_alpha1=MLPClassifier(solver="lbfgs", hidden_layer_sizes =[10, 10], activation='tanh', alpha=1,
                     max_iter=200, random_state=7)
mlp_alpha1.fit(X_train, y_train)

# 打分
print("training set:{:0.3f}".format( mlp_alpha1.score(X_train, y_train)) )
print("tesing set:{:0.3f}".format( mlp_alpha1.score(X_test, y_test)) )


# 画图
Z=mlp_alpha1.predict(np.c_[xx.ravel(), yy.ravel()])
Z=Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cmap_light, shading='auto')

# 散点图
plt.scatter(X[:, 0], X[:, 1], c=y, edgecolor="k", s=60)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("MLPClassifier:nodes=[10, 10], activation='tanh',alpha=1")
plt.show()

# 边界有更多细节了

/home/wangjl/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:549: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
  self.n_iter_ = _check_optimize_result("lbfgs", opt_res, self.max_iter)

training set:0.805
tesing set:0.889

手写数字识别 8x8（泛化差）

MNIST 数据集手写体数字识别，就像入门程序猿写 Hello world 一样，是非常基础的必修课。

载入数据

# https://h1ros.github.io/posts/loading-scikit-learns-mnist-dataset/
from sklearn.datasets import load_digits
mnist = load_digits()

print(mnist.keys())
print(mnist.data.shape, mnist.target.shape)
# data是1797行，每行64像素(8*8) 是一个图形每个像素的黑白值。
# target 是0-9的整型数字。

dict_keys(['data', 'target', 'frame', 'feature_names', 'target_names', 'images', 'DESCR'])
(1797, 64) (1797,)

import pandas as pd
print(pd.DataFrame(mnist.data).head())
pd.DataFrame(mnist.target).head()

    0    1    2     3     4     5    6    7    8    9   ...   54   55   56  \
0  0.0  0.0  5.0  13.0   9.0   1.0  0.0  0.0  0.0  0.0  ...  0.0  0.0  0.0   
1  0.0  0.0  0.0  12.0  13.0   5.0  0.0  0.0  0.0  0.0  ...  0.0  0.0  0.0   
2  0.0  0.0  0.0   4.0  15.0  12.0  0.0  0.0  0.0  0.0  ...  5.0  0.0  0.0   
3  0.0  0.0  7.0  15.0  13.0   1.0  0.0  0.0  0.0  8.0  ...  9.0  0.0  0.0   
4  0.0  0.0  0.0   1.0  11.0   0.0  0.0  0.0  0.0  0.0  ...  0.0  0.0  0.0   

    57   58    59    60    61   62   63  
0  0.0  6.0  13.0  10.0   0.0  0.0  0.0  
1  0.0  0.0  11.0  16.0  10.0  0.0  0.0  
2  0.0  0.0   3.0  11.0  16.0  9.0  0.0  
3  0.0  7.0  13.0  13.0   9.0  0.0  0.0  
4  0.0  0.0   2.0  16.0   4.0  0.0  0.0  

[5 rows x 64 columns]

	0
0	0
1	1
2	2
3	3
4	4

# 可视化图形
import matplotlib.pyplot as plt
plt.imshow(mnist.images[0]);

# 批量 可视化图形
fig, axes = plt.subplots(2, 10, figsize=(10, 2.8))
for i in range(20):
    axes[i//10, i %10].imshow(mnist.images[i], cmap='gray');
    axes[i//10, i %10].axis('off')
    axes[i//10, i %10].set_title(f"target: {mnist.target[i]}")
    
plt.tight_layout()

预处理

X=mnist.data/255
y=mnist.target

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=1200, test_size=500, random_state=62)

print(X_train.shape, X_test.shape)

(1200, 64) (500, 64)

训练 MLP 神经网络

from sklearn.neural_network import MLPClassifier
mlp_hw=MLPClassifier(solver="lbfgs", hidden_layer_sizes=[100, 100], max_iter=1000,
                    activation='relu', alpha=1e-5, random_state=62)

mlp_hw.fit(X_train, y_train)

# 打分
print("training set:{:0.3f}".format( mlp_hw.score(X_train, y_train)) )
print("tesing set:{:0.3f}".format( mlp_hw.score(X_test, y_test)) )

training set:1.000
tesing set:0.942

测试

import numpy as np

# 新建一个 8*8 画布，黑色字体，手写一个数字，保存。
from PIL import Image
image=Image.open("data/pic4.png").convert("F")
# 调整图像大小
image=image.resize( (8,8) )
arr=[]
for i in range(8):
    for j in range(8):
        pixel=1.0 - float(image.getpixel((j,i)))/255
        arr.append(pixel)
arr1=np.array(arr).reshape(1, -1)
plt.imshow( np.array(arr).reshape(8,8), cmap='gray' );
mlp_hw.predict(arr1)[0]

3

手写体识别 28x28

下载数据

# https://scikit-learn.org/stable/modules/preprocessing.html

# 下载数据
from sklearn.datasets import fetch_openml
X, y = fetch_openml("mnist_784", version=1, return_X_y=True, as_frame=False)
print(X.shape, y.shape)

(70000, 784) (70000,)

# 可视化
import numpy as np
import matplotlib.pyplot as plt
plt.imshow(X[0].reshape([28, 28]));

# 批量 可视化图形
fig, axes = plt.subplots(2, 10, figsize=(10, 2.8))
for i in range(20):
    axes[i//10, i %10].imshow(X[i].reshape([28, 28]), cmap='gray');
    axes[i//10, i %10].axis('off')
    axes[i//10, i %10].set_title(f"target: {y[i]}")
    
plt.tight_layout()

预处理

# 归一化到 0-1
X=X/255

# 拆分数据
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=5000, test_size=1000, random_state=62)
print(X_train.shape, X_test.shape)

(5000, 784) (1000, 784)

训练

from sklearn.neural_network import MLPClassifier
mlp_hw=MLPClassifier(solver="lbfgs", hidden_layer_sizes=[100, 100], max_iter=1000,
                    activation='relu', alpha=1e-5, random_state=62)

mlp_hw.fit(X_train, y_train)

# 打分
print("training set:{:0.3f}".format( mlp_hw.score(X_train, y_train)) )
print("tesing set:{:0.3f}".format( mlp_hw.score(X_test, y_test)) )

training set:1.000
tesing set:0.927

测试

import numpy as np

# 新建一个 28*28 画布，黑色字体，手写一个数字，保存。
from PIL import Image

def getNum(path="data/28_4.png"):
    fig=plt.figure(figsize=(2, 1))
    image=Image.open(path).convert("F")
    # 调整图像大小
    image=image.resize( (28,28) )
    arr=[]
    for i in range(28):
        for j in range(28):
            pixel=1.0 - float(image.getpixel((j,i)))/255
            arr.append(pixel)
    arr1=np.array(arr).reshape(1, -1)
    plt.imshow( np.array(arr).reshape(28,28), cmap='gray' );
    return mlp_hw.predict(arr1)[0]

getNum() #正确

'4'

getNum("data/28_6.png") #错

'5'

还很很傻，人眼能是别的，该网络不一定能识别