ニューラルネットワークチュートリアル(Keras編)

今日はこのサイトのNeural Networks Crash Tutorial – Keras version (ニューラルネットワーク集中チュートリアル-Keras編)をやる。チュートリアルを始める前に必要なデータをダンロードしてファイルを展開しておく。

!mkdir test1
%download https://btsd.ethz.ch/shareddata/BelgiumTSC/BelgiumTSC_Training.zip
Downloaded 'BelgiumTSC_Training.zip'.
%download https://btsd.ethz.ch/shareddata/BelgiumTSC/BelgiumTSC_Testing.zip
Downloaded 'BelgiumTSC_Testing.zip'.
!mkdir BelgiumTrafficSigns
cd BelgiumTrafficSigns
mv ../BelgiumTSC_Training.zip .
!unzip BelgiumTSC_Training.zip
!mv ../BelgiumTSC_Testing.zip .
!unzip BelgiumTSC_Testing.zip
cd ..

先ずはいくつかのお役立ち関数の作成から始める。

import os, skimage.data

def load_data(data_directory):
    directories = [d for d in os.listdir(data_directory) 
                   if os.path.isdir(os.path.join(data_directory, d))]
    labels = []
    images = []
    for d in directories:
        label_directory = os.path.join(data_directory, d)
        file_names = [os.path.join(label_directory, f) 
                      for f in os.listdir(label_directory) 
                      if f.endswith(".ppm")]
        for f in file_names:
            images.append(skimage.data.imread(f))
            labels.append(int(d))
    return images, labels
def plot_images(images, labels, color_map = "brg"):
    # Get the unique labels 
    unique_labels = set(labels)

    # Initialize the figure
    plt.figure(figsize=(20, 20))

    # Set a counter
    i = 1

    # For each unique label,
    for label in unique_labels:
        # You pick the first image for each label
        image = images[labels.index(label)]
        # Define 64 subplots 
        plt.subplot(8, 8, i)
        # Don't include axes
        plt.axis('off')
        # Add a title to each subplot 
        plt.title("Label {0} ({1})".format(label, labels.count(label)))
        # Add 1 to the counter
        i += 1
        # And you plot this first image 
        plt.imshow(image, cmap=color_map)
    
    # Show the plot
    plt.show()

Load training data

次に先程ダウンロードした訓練データをロードする。

train_data_directory = "./BelgiumTrafficSigns/Training"
images, labels = load_data(train_data_directory)

画像がどのように分布しているのかを見るためにラベルのヒストグラムをプロットする。

# Import the `pyplot` module of matplotlib
import matplotlib.pyplot as plt 
plt.rcParams['figure.figsize'] = 16, 8
plt.rcParams["font.size"] = "17"
# Make a histogram with 62 bins of the `labels` data
plt.hist(labels, 62)

# Show the plot
plt.show()

いくつかの画像とそれらの形を無作為抽出して確認する。

import random

# Determine the (random) indexes of the images that you want to see 
traffic_signs = random.sample(range(len(images)), 4)

# Fill out the subplots with the random images and add shape
for i in range(len(traffic_signs)):
    plt.subplot(1, 4, i+1)
    plt.axis('off')
    plt.imshow(images[traffic_signs[i]])
    plt.subplots_adjust(wspace=0.5)
    plt.show()
    print("shape: {0}".format(images[traffic_signs[i]].shape))
shape: (82, 53, 3)
shape: (64, 69, 3)
shape: (95, 91, 3)
shape: (499, 517, 3)

各カテゴリーの最初のサンプルをプロットする。

plt.rcParams["font.size"] = "14"
plot_images(images,labels)

Pre-process images

画像を全て同じサイズと解像度にする。

# Import the `transform` module from `skimage`
from skimage import transform 

# set desired number of pixels
px = 32

# Rescale the images in the `images` array
simages = [transform.resize(image, (px, px), mode='constant') for image in images]
/root/.pyenv/versions/py365/lib/python3.6/site-packages/skimage/transform/_warps.py:110: UserWarning: Anti-aliasing will be enabled by default in skimage 0.15 to avoid aliasing artifacts when down-sampling images.
  warn("Anti-aliasing will be enabled by default in skimage 0.15 to "
plot_images(simages,labels)

画像をグレイスケールに変換する。

# Import `rgb2gray` from `skimage.color`
from skimage.color import rgb2gray

# Convert `simages` to grayscale
simagesg = [rgb2gray(img) for img in simages]
plot_images(simagesg, labels, color_map="gray")
# Compare image sizes
print('original = {}, scaled = {}, grayscale = {}'.format(images[0].shape,simages[0].shape,simagesg[0].shape))
original = (73, 79, 3), scaled = (32, 32, 3), grayscale = (32, 32)

Create the neural network model

import keras 
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten, Dropout
Using TensorFlow backend.
import numpy as np

# Convert labels to categorical one-hot encoding
one_hot_labels = keras.utils.to_categorical(labels, num_classes=62)

# Convert images to numpy arrays
data = np.array(simagesg)

data.shape
(4575, 32, 32)
gmodel = Sequential()

gmodel.add(Flatten(input_shape=data.shape[1:]))
gmodel.add(Dense(62, activation='relu'))
gmodel.add(Dropout(0.5))
gmodel.add(Dense(62, activation='softmax'))

Train the model

# Multi-class classification problem
gmodel.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
import tensorflow as tf
session_config = tf.ConfigProto()
session_config.gpu_options.per_process_gpu_memory_fraction = 0.5
# Train the model, iterating on the data in batches of 32 samples
gmodel.fit(data, one_hot_labels, epochs=10, batch_size=32, verbose=1)
Epoch 1/10
4575/4575 [==============================] - 1s 193us/step - loss: 3.6199 - acc: 0.1519
Epoch 2/10
4575/4575 [==============================] - 0s 63us/step - loss: 3.0518 - acc: 0.2540
Epoch 3/10
4575/4575 [==============================] - 0s 58us/step - loss: 2.7741 - acc: 0.3078
Epoch 4/10
4575/4575 [==============================] - 0s 62us/step - loss: 2.5733 - acc: 0.3513
Epoch 5/10
4575/4575 [==============================] - 0s 60us/step - loss: 2.4135 - acc: 0.3781
Epoch 6/10
4575/4575 [==============================] - 0s 61us/step - loss: 2.2981 - acc: 0.4052
Epoch 7/10
4575/4575 [==============================] - 0s 57us/step - loss: 2.1825 - acc: 0.4142
Epoch 8/10
4575/4575 [==============================] - 0s 56us/step - loss: 2.0984 - acc: 0.4310
Epoch 9/10
4575/4575 [==============================] - 0s 62us/step - loss: 2.0256 - acc: 0.4380
Epoch 10/10
4575/4575 [==============================] - 0s 57us/step - loss: 1.9714 - acc: 0.4570
<keras.callbacks.History at 0x7f412ccfa8d0>

Evaluate performance on some samples

ここでの目標は訓練がうまくいったかを迅速に評価すること。

完全なモデル検証は、ただ単に訓練データ・セットの標本をいくつか評価する以上のことが要求される。

N = 10
sample_indexes = random.sample(range(len(data)), N)
sample_images = np.array([data[i] for i in sample_indexes])
sample_labels = np.array([labels[i] for i in sample_indexes])

# Run the "correct_pred" operation
predicted = np.array(gmodel.predict_classes(sample_images))
      
error, = np.nonzero(sample_labels - predicted)
accuracy = 100*(1-error.size/N)
    
# Print the real and predicted labels
print("Truth:    ", sample_labels)
print("Predicted:", predicted, "accuracy = {}%".format(accuracy))
Truth:     [61 32 38 12 22 22 38  9 57 31]
Predicted: [61 32 38  7 22 22 38  1 56 31] accuracy = 70.0%
# Display the predictions and the ground truth visually.
fig = plt.figure(figsize=(px, px))
j = 1
for i in range(len(sample_images)):
    
    truth = sample_labels[i]
    prediction = predicted[i]
    
    predicted_image = data[labels.index(prediction)]

    plt.subplot(10, 2, j)
    plt.axis('off')
    color='green' if truth == prediction else 'red'
    plt.text(40, 10, "Truth:        {0}\nPrediction: {1}".format(truth, prediction), fontsize=18, color=color)
    plt.imshow(sample_images[i], cmap="gray")

    j += 1
    
    plt.subplot(10, 2, j)
    plt.axis('off')
    plt.imshow(predicted_image, cmap="gray")

    j += 1
    
# Show the plot
plt.show()

Can color improve the results?

今度はカラー画像を使ってプロセスを繰り返す。

import numpy as np

# Convert labels to categorical one-hot encoding
one_hot_labels = keras.utils.to_categorical(labels, num_classes=62)

# Convert images to numpy arrays
data = np.array(simages)

data.shape
(4575, 32, 32, 3)
cmodel = Sequential()

cmodel.add(Flatten(input_shape=data.shape[1:]))
cmodel.add(Dense(62, activation='relu'))
cmodel.add(Dropout(0.5))
cmodel.add(Dense(62, activation='softmax'))

Train the model

# Multi-class classification problem
cmodel.compile(optimizer='adam',
               loss='categorical_crossentropy',
               metrics=['accuracy'])
session_config = tf.ConfigProto()
session_config.gpu_options.per_process_gpu_memory_fraction = 0.5
# Train the model, iterating on the data in batches of 32 samples
cmodel.fit(data, one_hot_labels, epochs=10, batch_size=32, verbose=1)
Epoch 1/10
4575/4575 [==============================] - 0s 106us/step - loss: 3.6547 - acc: 0.1342
Epoch 2/10
4575/4575 [==============================] - 0s 65us/step - loss: 3.1277 - acc: 0.2254
Epoch 3/10
4575/4575 [==============================] - 0s 66us/step - loss: 2.8765 - acc: 0.2505
Epoch 4/10
4575/4575 [==============================] - 0s 70us/step - loss: 2.7201 - acc: 0.2745
Epoch 5/10
4575/4575 [==============================] - 0s 68us/step - loss: 2.5835 - acc: 0.2861
Epoch 6/10
4575/4575 [==============================] - 0s 67us/step - loss: 2.4797 - acc: 0.3082
Epoch 7/10
4575/4575 [==============================] - 0s 66us/step - loss: 2.3838 - acc: 0.3178
Epoch 8/10
4575/4575 [==============================] - 0s 70us/step - loss: 2.3441 - acc: 0.3237
Epoch 9/10
4575/4575 [==============================] - 0s 65us/step - loss: 2.2782 - acc: 0.3458
Epoch 10/10
4575/4575 [==============================] - 0s 71us/step - loss: 2.2228 - acc: 0.3412
<keras.callbacks.History at 0x7f40cc1e9b70>

Evaluate performance on some samples

N = 10
sample_indexes = random.sample(range(len(data)), N)
sample_gimages = np.array([simagesg[i] for i in sample_indexes])
sample_cimages = np.array([simages[i] for i in sample_indexes])
sample_labels = np.array([labels[i] for i in sample_indexes])

# Run the "correct_pred" operation
gpredicted = gmodel.predict_classes(sample_gimages)
cpredicted = cmodel.predict_classes(sample_cimages)

gerror, = np.nonzero(sample_labels - gpredicted)
cerror, = np.nonzero(sample_labels - cpredicted)

gaccuracy = 100*(1-gerror.size/N)
caccuracy = 100*(1-cerror.size/N)
    
# Print the real and predicted labels
print("Truth:", sample_labels)
print("GS:   ", gpredicted, "accuracy = {}%".format(gaccuracy))
print("Color:", cpredicted, "accuracy = {}%".format(caccuracy))
Truth: [39 32  1 61 40 28 37 28 13 41]
GS:    [38 32  1 61 40 28 37 32 13 40] accuracy = 70.0%
Color: [38 32  1 61 40 28 38 32  7 40] accuracy = 50.0%
# Display the predictions and the ground truth visually.
#fig = plt.figure(figsize=(10, 10))
fig = plt.figure(figsize=(28,28))
j = 1
for i in range(len(sample_images)):
    
    truth = sample_labels[i]
    gprediction = gpredicted[i]
    cprediction = cpredicted[i]
    
    predicted_gimage = simagesg[labels.index(gprediction)]
    predicted_cimage = simages[labels.index(cprediction)]

    plt.subplot(10, 3, j)
    plt.axis('off')
    plt.text(40, 10, "Truth:            {}".format(truth), 
             fontsize=18)
    color='green' if truth == gprediction else 'red'
    plt.text(40, 15, "GS Prediction:    {}".format(gprediction), 
             fontsize=18, color=color)
    color='green' if truth == cprediction else 'red'
    plt.text(40, 20, "Color Prediction: {}".format(cprediction), 
             fontsize=18, color=color)
    plt.imshow(sample_cimages[i])

    j += 1
    
    plt.subplot(10, 3, j)
    plt.axis('off')
    plt.imshow(predicted_gimage, cmap="gray")

    j += 1

    plt.subplot(10, 3, j)
    plt.axis('off')
    plt.imshow(predicted_cimage)

    j += 1
    
# Show the plot
plt.show()

Can features improve the results?

今度は、深層学習で行われることに似た方法でニューラルネットワークに色々な層を足す。

import numpy as np

# Convert labels to categorical one-hot encoding
one_hot_labels = keras.utils.to_categorical(labels, num_classes=62)

# Convert images to numpy arrays
data = np.array(simages)

data.shape
(4575, 32, 32, 3)
from keras.layers import Flatten, Conv2D, MaxPooling2D

fmodel = Sequential()

# Convolutional neural network to extract features
fmodel.add(Conv2D(32, (3, 3), input_shape=data.shape[1:], activation='relu'))
fmodel.add(MaxPooling2D(pool_size=(2, 2)))

fmodel.add(Conv2D(32, (3, 3), activation='relu'))
fmodel.add(MaxPooling2D(pool_size=(2, 2)))

fmodel.add(Conv2D(32, (3, 3), activation='relu'))
fmodel.add(MaxPooling2D(pool_size=(2, 2)))

# Converts our 3D feature maps to 1D feature vectors
fmodel.add(Flatten())

# Classifier network
fmodel.add(Dense(62, activation='relu'))
fmodel.add(Dropout(0.5))
fmodel.add(Dense(62, activation='softmax'))

Train the model

# Multi-class classification problem
fmodel.compile(optimizer='adam',
               loss='categorical_crossentropy',
               metrics=['accuracy'])
session_config = tf.ConfigProto()
session_config.gpu_options.per_process_gpu_memory_fraction = 0.5
# Train the model, iterating on the data in batches of 32 samples
fmodel.fit(data, one_hot_labels, epochs=10, batch_size=32, verbose=1)
Epoch 1/10
4575/4575 [==============================] - 1s 302us/step - loss: 3.5855 - acc: 0.1489
Epoch 2/10
4575/4575 [==============================] - 0s 104us/step - loss: 2.5093 - acc: 0.3880
Epoch 3/10
4575/4575 [==============================] - 1s 115us/step - loss: 1.8063 - acc: 0.5353
Epoch 4/10
4575/4575 [==============================] - 0s 108us/step - loss: 1.4099 - acc: 0.6208
Epoch 5/10
4575/4575 [==============================] - 1s 117us/step - loss: 1.1482 - acc: 0.6918
Epoch 6/10
4575/4575 [==============================] - 1s 115us/step - loss: 0.9808 - acc: 0.7235
Epoch 7/10
4575/4575 [==============================] - 1s 114us/step - loss: 0.8503 - acc: 0.7561
Epoch 8/10
4575/4575 [==============================] - 1s 112us/step - loss: 0.7530 - acc: 0.7742
Epoch 9/10
4575/4575 [==============================] - 1s 115us/step - loss: 0.7062 - acc: 0.7858
Epoch 10/10
4575/4575 [==============================] - 1s 111us/step - loss: 0.5912 - acc: 0.8208
<keras.callbacks.History at 0x7f40f471bb00>

Evaluate performance on some samples

sample_indexes = random.sample(range(len(simages)), 10)
sample_gimages = np.array([simagesg[i] for i in sample_indexes])
sample_cimages = np.array([simages[i] for i in sample_indexes])
sample_labels = np.array([labels[i] for i in sample_indexes])

# Run the "correct_pred" operation
gpredicted = gmodel.predict_classes(sample_gimages)
cpredicted = cmodel.predict_classes(sample_cimages)
fpredicted = fmodel.predict_classes(sample_cimages)

gerror, = np.nonzero(sample_labels - gpredicted)
cerror, = np.nonzero(sample_labels - cpredicted)
ferror, = np.nonzero(sample_labels - fpredicted)

gaccuracy = 100*(1-gerror.size/N)
caccuracy = 100*(1-cerror.size/N)
faccuracy = 100*(1-ferror.size/N)
    
# Print the real and predicted labels
print("Truth:", sample_labels)
print("GS:   ", gpredicted, "accuracy = {}%".format(gaccuracy))
print("Color:", cpredicted, "accuracy = {}%".format(caccuracy))
print("DL:   ", fpredicted, "accuracy = {}%".format(faccuracy))
Truth: [47 57 32  3 38 19 42 32 32 32]
GS:    [47 56 32  7 38 19 38 32 32 32] accuracy = 70.0%
Color: [47 57 32 13 38 19 40 32 32 32] accuracy = 80.0%
DL:    [47 57 32 14 38 19 42 32 32 32] accuracy = 90.0%
# Display the predictions and the ground truth visually.
#fig = plt.figure(figsize=(10, 10))
fig = plt.figure(figsize=(28,28))
j = 1
for i in range(len(sample_images)):
    
    truth = sample_labels[i]
    gprediction = gpredicted[i]
    cprediction = cpredicted[i]
    fprediction = fpredicted[i]
    
    predicted_gimage = simagesg[labels.index(gprediction)]
    predicted_cimage = simages[labels.index(cprediction)]
    predicted_fimage = simages[labels.index(fprediction)]

    plt.subplot(10, 4, j)
    plt.axis('off')
    plt.text(40, 10, "Truth:            {}".format(truth), 
             fontsize=18)
    color='green' if truth == gprediction else 'red'
    plt.text(40, 15, "GS Prediction:    {}".format(gprediction), 
             fontsize=18, color=color)
    color='green' if truth == cprediction else 'red'
    plt.text(40, 20, "Color Prediction: {}".format(cprediction), 
             fontsize=18, color=color)
    color='green' if truth == fprediction else 'red'
    plt.text(40, 25, "DL Prediction:    {}".format(fprediction), 
             fontsize=18, color=color)
    plt.imshow(sample_cimages[i])

    j += 1
    
    plt.subplot(10, 4, j)
    plt.axis('off')
    plt.imshow(predicted_gimage, cmap="gray")

    j += 1

    plt.subplot(10, 4, j)
    plt.axis('off')
    plt.imshow(predicted_cimage)

    j += 1

    plt.subplot(10, 4, j)
    plt.axis('off')
    plt.imshow(predicted_fimage)

    j += 1
    
# Show the plot
plt.show()

下のコードは、最初の畳み込み層が処理される後に起こることをそのまま視覚化する。

可視化したい層をインデックスkを変更することで選択できる。

from keras.models import Model

k = 0
intermediate_layer_model = Model(inputs=fmodel.input,
                                 outputs=fmodel.layers[k].output)
intermediate_output = intermediate_layer_model.predict(sample_cimages)
print(intermediate_output.shape)
(10, 30, 30, 32)

stacked image(積層画像)として層を可視化する。

import math
M = math.floor(intermediate_output.shape[-1]/3)
N = len(sample_cimages)
print('M = {}, N = {}'.format(M,N))

fig = plt.figure(figsize=(px,px))
for i, img in enumerate(sample_cimages):
    plt.subplot(N,M+1,i*(M+1)+1)
    plt.imshow(img)
    plt.axis('off')
    for j in range(M):
        #print('i = {}, j = {}, i*(M+1)+j+2 = {}'.format(i,j,i*(M+1)+j+2))
        plt.subplot(N,M+1,i*(M+1)+j+2)
        plt.imshow((intermediate_output[i,:,:,j:j+3]*255).astype(np.uint8))
        plt.axis('off')
plt.show()
M = 10, N = 10

flattened image(平坦化画像)として層を視覚化する。

K = intermediate_output.shape[3]
M = np.prod(intermediate_output.shape[1:3])
N = len(sample_cimages)
# print('M = {}, N = {}, K = {}'.format(M,N,K))

fig = plt.figure(figsize=(px,px))
for i, img in enumerate(sample_cimages):
    plt.subplot(N,2,i*2+1)
    plt.imshow(img)
    plt.axis('off')

    plt.subplot(N,2,i*2+2)
    fimg = np.reshape(intermediate_output[i,:,:,:], [M, K], order = 'C').transpose()
    plt.imshow(fimg, cmap='gray')
    plt.axis('off')
plt.show()
!pip3 install --upgrade matplotlib
Collecting matplotlib
  Downloading https://files.pythonhosted.org/packages/9e/59/f235ab21bbe7b7c6570c4abf17ffb893071f4fa3b9cf557b09b60359ad9a/matplotlib-2.2.3-cp36-cp36m-manylinux1_x86_64.whl (12.6MB)
    100% |################################| 12.6MB 6.1MB/s  eta 0:00:01
Requirement already satisfied, skipping upgrade: pytz in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from matplotlib) (2018.4)
Requirement already satisfied, skipping upgrade: six>=1.10 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from matplotlib) (1.11.0)
Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from matplotlib) (1.0.1)
Requirement already satisfied, skipping upgrade: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from matplotlib) (2.2.0)
Requirement already satisfied, skipping upgrade: cycler>=0.10 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from matplotlib) (0.10.0)
Requirement already satisfied, skipping upgrade: python-dateutil>=2.1 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from matplotlib) (2.7.3)
Requirement already satisfied, skipping upgrade: numpy>=1.7.1 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from matplotlib) (1.14.4)
Requirement already satisfied, skipping upgrade: setuptools in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib) (39.0.1)
Installing collected packages: matplotlib
  Found existing installation: matplotlib 2.2.2
    Uninstalling matplotlib-2.2.2:
      Successfully uninstalled matplotlib-2.2.2
Successfully installed matplotlib-2.2.3