# CS231n/assignment2/Fully-Connected Neural Nets

スポンサーリンク

## Fully-Connected Neural Nets¶

def layer_forward(x, w):
""" Receive inputs x and weights w """
# Do some computations ...
z = # ... some intermediate value
# Do some more computations ...
out = # the output
cache = (x, w, z, out) # Values we need to compute gradients
return out, cache


バックワードパスはアップストリームデリバティブとキャッシュオブジェクトを受け取って、インプットと重みに対応するグラディエントを返す。

def layer_backward(dout, cache):
"""
Receive dout (derivative of loss with respect to outputs) and cache,
and compute derivative with respect to inputs.
"""
# Unpack cache values
x, w, z, out = cache
# Use values in cache to compute derivatives
dx = # Derivative of loss with respect to x
dw = # Derivative of loss with respect to w
return dx, dw


# As usual, a bit of setup
from __future__ import print_function
import time
import numpy as np
import matplotlib.pyplot as plt
from cs231n.classifiers.fc_net import *
from cs231n.data_utils import get_CIFAR10_data
from cs231n.solver import Solver

plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

def rel_error(x, y):
""" returns relative error """
return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))
%matplotlib inline

# Load the (preprocessed) CIFAR10 data.
data = get_CIFAR10_data()
for k, v in list(data.items()):
print(('%s: ' % k, v.shape))

('X_train: ', (49000, 3, 32, 32))
('y_train: ', (49000,))
('X_val: ', (1000, 3, 32, 32))
('y_val: ', (1000,))
('X_test: ', (1000, 3, 32, 32))
('y_test: ', (1000,))


## Affine layer: foward¶

cs231n/layers.pyを開いて、affine_forward関数を実装し、終わったら下記のコードを実行して実装をテストする。

# Test the affine_forward function
num_inputs = 2
input_shape = (4, 5, 6)
output_dim = 3
input_size = num_inputs * np.prod(input_shape)
weight_size = output_dim * np.prod(input_shape)
x = np.linspace(-0.1, 0.5, num=input_size).reshape(num_inputs, *input_shape)
w = np.linspace(-0.2, 0.3, num=weight_size).reshape(np.prod(input_shape), output_dim)
b = np.linspace(-0.3, 0.1, num=output_dim)
out, _ = affine_forward(x, w, b)
correct_out = np.array([[ 1.49834967,  1.70660132,  1.91485297],
[ 3.25553199,  3.5141327,   3.77273342]])
# Compare your output with ours. The error should be around 1e-9.
print('Testing affine_forward function:')
print('difference: ', rel_error(out, correct_out))

Testing affine_forward function:
difference:  9.769848888397517e-10


## Affine layer: backward¶

# Test the affine_backward function
np.random.seed(231)
x = np.random.randn(10, 2, 3)
w = np.random.randn(6, 5)
b = np.random.randn(5)
dout = np.random.randn(10, 5)
dx_num = eval_numerical_gradient_array(lambda x: affine_forward(x, w, b)[0], x, dout)
dw_num = eval_numerical_gradient_array(lambda w: affine_forward(x, w, b)[0], w, dout)
db_num = eval_numerical_gradient_array(lambda b: affine_forward(x, w, b)[0], b, dout)
_, cache = affine_forward(x, w, b)
dx, dw, db = affine_backward(dout, cache)
# The error should be around 1e-10
print('Testing affine_backward function:')
print('dx error: ', rel_error(dx_num, dx))
print('dw error: ', rel_error(dw_num, dw))
print('db error: ', rel_error(db_num, db))

Testing affine_backward function:
dx error:  1.0908199508708189e-10
dw error:  2.1752635504596857e-10
db error:  7.736978834487815e-12


## ReLU layer: forward¶

ReLU activation function(活性化関数)用のフォワードパスをrelu_forward function内に実装して、下記のコードを使って実装をテストする。

# Test the relu_forward function
x = np.linspace(-0.5, 0.5, num=12).reshape(3, 4)
out, _ = relu_forward(x)
correct_out = np.array([[ 0.,          0.,          0.,          0.,        ],
[ 0.,          0.,          0.04545455,  0.13636364,],
[ 0.22727273,  0.31818182,  0.40909091,  0.5,       ]])
# Compare your output with ours. The error should be around 5e-8
print('Testing relu_forward function:')
print('difference: ', rel_error(out, correct_out))

Testing relu_forward function:
difference:  4.999999798022158e-08


## ReLU layer: backward¶

np.random.seed(231)
x = np.random.randn(10, 10)
dout = np.random.randn(*x.shape)
dx_num = eval_numerical_gradient_array(lambda x: relu_forward(x)[0], x, dout)
_, cache = relu_forward(x)
dx = relu_backward(dout, cache)
# The error should be around 3e-12
print('Testing relu_backward function:')
print('dx error: ', rel_error(dx_num, dx))

Testing relu_backward function:
dx error:  3.2756349136310288e-12


## “Sandwich” layers¶

ニューラルネットでよく使われるいくつかの一般的なレイヤーパターンがある。例えば、affine layer(アフィン層)はReLU nonlinearity(非線形)がよく後に続く。これらの一般的なパターンを簡素化する目的で、cs231n/layer_utils.pyにいくつかの利便性の高い層を定義する。とりあえずaffine_relu_forward/backward関数に着目し、バックワードパスを数値的に勾配チェックするのに下記を実行する。

from cs231n.layer_utils import affine_relu_forward, affine_relu_backward
np.random.seed(231)
x = np.random.randn(2, 3, 4)
w = np.random.randn(12, 10)
b = np.random.randn(10)
dout = np.random.randn(2, 10)
out, cache = affine_relu_forward(x, w, b)
dx, dw, db = affine_relu_backward(dout, cache)
dx_num = eval_numerical_gradient_array(lambda x: affine_relu_forward(x, w, b)[0], x, dout)
dw_num = eval_numerical_gradient_array(lambda w: affine_relu_forward(x, w, b)[0], w, dout)
db_num = eval_numerical_gradient_array(lambda b: affine_relu_forward(x, w, b)[0], b, dout)
print('Testing affine_relu_forward:')
print('dx error: ', rel_error(dx_num, dx))
print('dw error: ', rel_error(dw_num, dw))
print('db error: ', rel_error(db_num, db))

Testing affine_relu_forward:
dx error:  6.395535042049294e-11
dw error:  8.162015570444288e-11
db error:  7.826724021458994e-12


## Loss layers: Softmax and SVM¶

np.random.seed(231)
num_classes, num_inputs = 10, 50
x = 0.001 * np.random.randn(num_inputs, num_classes)
y = np.random.randint(num_classes, size=num_inputs)
dx_num = eval_numerical_gradient(lambda x: svm_loss(x, y)[0], x, verbose=False)
loss, dx = svm_loss(x, y)
# Test svm_loss function. Loss should be around 9 and dx error should be 1e-9
print('Testing svm_loss:')
print('loss: ', loss)
print('dx error: ', rel_error(dx_num, dx))
dx_num = eval_numerical_gradient(lambda x: softmax_loss(x, y)[0], x, verbose=False)
loss, dx = softmax_loss(x, y)
# Test softmax_loss function. Loss should be 2.3 and dx error should be 1e-8
print('Testing softmax_loss:')
print('loss: ', loss)
print('dx error: ', rel_error(dx_num, dx))

Testing svm_loss:
loss:  8.999602749096233
dx error:  1.4021566006651672e-09
Testing softmax_loss:
loss:  2.302545844500738
dx error:  9.384673161989355e-09


## Two-layer network¶

np.random.seed(231)
N, D, H, C = 3, 5, 50, 7
X = np.random.randn(N, D)
y = np.random.randint(C, size=N)
std = 1e-3
model = TwoLayerNet(input_dim=D, hidden_dim=H, num_classes=C, weight_scale=std)
print('Testing initialization ... ')
W1_std = abs(model.params['W1'].std() - std)
b1 = model.params['b1']
W2_std = abs(model.params['W2'].std() - std)
b2 = model.params['b2']
assert W1_std < std / 10, 'First layer weights do not seem right'
assert np.all(b1 == 0), 'First layer biases do not seem right'
assert W2_std < std / 10, 'Second layer weights do not seem right'
assert np.all(b2 == 0), 'Second layer biases do not seem right'
print('Testing test-time forward pass ... ')
model.params['W1'] = np.linspace(-0.7, 0.3, num=D*H).reshape(D, H)
model.params['b1'] = np.linspace(-0.1, 0.9, num=H)
model.params['W2'] = np.linspace(-0.3, 0.4, num=H*C).reshape(H, C)
model.params['b2'] = np.linspace(-0.9, 0.1, num=C)
X = np.linspace(-5.5, 4.5, num=N*D).reshape(D, N).T
scores = model.loss(X)
correct_scores = np.asarray(
[[11.53165108,  12.2917344,   13.05181771,  13.81190102,  14.57198434, 15.33206765,  16.09215096],
[12.05769098,  12.74614105,  13.43459113,  14.1230412,   14.81149128, 15.49994135,  16.18839143],
[12.58373087,  13.20054771,  13.81736455,  14.43418138,  15.05099822, 15.66781506,  16.2846319 ]])
scores_diff = np.abs(scores - correct_scores).sum()
assert scores_diff < 1e-6, 'Problem with test-time forward pass'
print('Testing training loss (no regularization)')
y = np.asarray([0, 5, 1])
correct_loss = 3.4702243556
assert abs(loss - correct_loss) < 1e-10, 'Problem with training-time loss'
model.reg = 1.0
correct_loss = 26.5948426952
assert abs(loss - correct_loss) < 1e-10, 'Problem with regularization loss'
for reg in [0.0, 0.7]:
print('Running numeric gradient check with reg = ', reg)
model.reg = reg
f = lambda _: model.loss(X, y)[0]

Testing initialization ...
Testing test-time forward pass ...
Testing training loss (no regularization)
Running numeric gradient check with reg =  0.0
W1 relative error: 1.83e-08
W2 relative error: 3.37e-10
b1 relative error: 8.01e-09
b2 relative error: 4.33e-10
Running numeric gradient check with reg =  0.7
W1 relative error: 2.53e-07
W2 relative error: 7.98e-08
b1 relative error: 1.35e-08
b2 relative error: 1.97e-09


## Solver¶

##############################################################################
# TODO: Use a Solver instance to train a TwoLayerNet that achieves at least  #
# 50% accuracy on the validation set.                                        #
##############################################################################
model = TwoLayerNet()
solver = Solver(model, data,
update_rule='sgd',
optim_config={
'learning_rate': 1e-3,
},
lr_decay=0.95,
num_epochs=10, batch_size=236,
print_every=100)
solver.train()
##############################################################################
#                             END OF YOUR CODE                               #
##############################################################################

(Iteration 1 / 2070) loss: 2.305961
(Epoch 0 / 10) train acc: 0.113000; val_acc: 0.084000
(Iteration 101 / 2070) loss: 1.782752
(Iteration 201 / 2070) loss: 1.660997
(Epoch 1 / 10) train acc: 0.411000; val_acc: 0.413000
(Iteration 301 / 2070) loss: 1.623456
(Iteration 401 / 2070) loss: 1.580220
(Epoch 2 / 10) train acc: 0.485000; val_acc: 0.455000
(Iteration 501 / 2070) loss: 1.516204
(Iteration 601 / 2070) loss: 1.451538
(Epoch 3 / 10) train acc: 0.488000; val_acc: 0.468000
(Iteration 701 / 2070) loss: 1.622765
(Iteration 801 / 2070) loss: 1.488118
(Epoch 4 / 10) train acc: 0.514000; val_acc: 0.491000
(Iteration 901 / 2070) loss: 1.405215
(Iteration 1001 / 2070) loss: 1.423761
(Epoch 5 / 10) train acc: 0.505000; val_acc: 0.484000
(Iteration 1101 / 2070) loss: 1.345751
(Iteration 1201 / 2070) loss: 1.259718
(Epoch 6 / 10) train acc: 0.585000; val_acc: 0.499000
(Iteration 1301 / 2070) loss: 1.333484
(Iteration 1401 / 2070) loss: 1.344973
(Epoch 7 / 10) train acc: 0.581000; val_acc: 0.500000
(Iteration 1501 / 2070) loss: 1.315636
(Iteration 1601 / 2070) loss: 1.407963
(Epoch 8 / 10) train acc: 0.568000; val_acc: 0.498000
(Iteration 1701 / 2070) loss: 1.173208
(Iteration 1801 / 2070) loss: 1.253118
(Epoch 9 / 10) train acc: 0.582000; val_acc: 0.507000
(Iteration 1901 / 2070) loss: 1.245580
(Iteration 2001 / 2070) loss: 1.303172
(Epoch 10 / 10) train acc: 0.582000; val_acc: 0.521000

# Run this cell to visualize training loss and train / val accuracy
#plt.rcParams['figure.figsize'] = 10, 10
plt.rcParams["font.size"] = "20"
plt.subplot(2, 1, 1)
plt.title('Training loss')
plt.plot(solver.loss_history, 'o')
plt.xlabel('Iteration')

plt.subplot(2, 1, 2)
plt.title('Accuracy')
plt.plot(solver.train_acc_history, '-o', label='train')
plt.plot(solver.val_acc_history, '-o', label='val')
plt.plot([0.5] * len(solver.val_acc_history), 'k--')
plt.xlabel('Epoch')
plt.legend(loc='lower right')
plt.gcf().set_size_inches(23, 18)
plt.show()


## Initial loss and gradient check¶

サニティーチェックとして、正則化有り/無しの両方でネットワークの初期損失チェックと勾配チェックをするために下記を実行する。初期損失は妥当だと思えるか？勾配チェッキングでは、エラーは大体1e-6以下になるはず。

np.random.seed(251)
N, D, H1, H2, C = 2, 15, 20, 30, 10
X = np.random.randn(N, D)
y = np.random.randint(C, size=(N,))
for reg in [0, 3.14]:
print('Running check with reg = ', reg)
model = FullyConnectedNet([H1, H2], input_dim=D, num_classes=C,
reg=reg, weight_scale=5e-2, dtype=np.float64)
print('Initial loss: ', loss)
f = lambda _: model.loss(X, y)[0]

Running check with reg =  0
Initial loss:  2.307941433441618
W1 relative error: 4.18e-07
W2 relative error: 6.68e-07
W3 relative error: 3.33e-07
b1 relative error: 2.57e-08
b2 relative error: 7.53e-09
b3 relative error: 6.71e-11
Running check with reg =  3.14
Initial loss:  6.765122333792326
W1 relative error: 1.44e-08
W2 relative error: 2.36e-08
W3 relative error: 4.82e-07
b1 relative error: 5.47e-08
b2 relative error: 4.61e-09
b3 relative error: 1.08e-10


もう一つのサニティーチェックとして、50画像の小さなデータセットを過学習することができることを確かめる。最初に、各隠れ層が100ユニットを持った3層ネットワークを試してみる。学習率と初期化スケールの微調整が必要になるが、過学習ができて20エポック以内で100％の訓練精度を達成できるはず。

# TODO: Use a three-layer Net to overfit 50 training examples.
num_train = 50
small_data = {
'X_train': data['X_train'][:num_train],
'y_train': data['y_train'][:num_train],
'X_val': data['X_val'],
'y_val': data['y_val'],
}
weight_scale = 1e-2
learning_rate = 1e-2
model = FullyConnectedNet([100, 100],
weight_scale=weight_scale, dtype=np.float64)
solver = Solver(model, small_data,
print_every=10, num_epochs=20, batch_size=25,
update_rule='sgd',
optim_config={
'learning_rate': learning_rate,
}
)
solver.train()
plt.rcParams['figure.figsize'] = 20, 10
plt.plot(solver.loss_history, 'o', markersize=10)
plt.title('Training loss history')
plt.xlabel('Iteration')
plt.ylabel('Training loss')
plt.show()

(Iteration 1 / 40) loss: 2.348306
(Epoch 0 / 20) train acc: 0.280000; val_acc: 0.098000
(Epoch 1 / 20) train acc: 0.320000; val_acc: 0.121000
(Epoch 2 / 20) train acc: 0.380000; val_acc: 0.130000
(Epoch 3 / 20) train acc: 0.480000; val_acc: 0.158000
(Epoch 4 / 20) train acc: 0.580000; val_acc: 0.171000
(Epoch 5 / 20) train acc: 0.560000; val_acc: 0.131000
(Iteration 11 / 40) loss: 1.395577
(Epoch 6 / 20) train acc: 0.700000; val_acc: 0.153000
(Epoch 7 / 20) train acc: 0.840000; val_acc: 0.160000
(Epoch 8 / 20) train acc: 0.880000; val_acc: 0.158000
(Epoch 9 / 20) train acc: 0.840000; val_acc: 0.213000
(Epoch 10 / 20) train acc: 0.880000; val_acc: 0.192000
(Iteration 21 / 40) loss: 0.610800
(Epoch 11 / 20) train acc: 0.940000; val_acc: 0.173000
(Epoch 12 / 20) train acc: 1.000000; val_acc: 0.177000
(Epoch 13 / 20) train acc: 1.000000; val_acc: 0.185000
(Epoch 14 / 20) train acc: 1.000000; val_acc: 0.186000
(Epoch 15 / 20) train acc: 0.980000; val_acc: 0.191000
(Iteration 31 / 40) loss: 0.064014
(Epoch 16 / 20) train acc: 1.000000; val_acc: 0.196000
(Epoch 17 / 20) train acc: 1.000000; val_acc: 0.172000
(Epoch 18 / 20) train acc: 1.000000; val_acc: 0.171000
(Epoch 19 / 20) train acc: 1.000000; val_acc: 0.175000
(Epoch 20 / 20) train acc: 1.000000; val_acc: 0.190000


# TODO: Use a five-layer Net to overfit 50 training examples.
num_train = 50
small_data = {
'X_train': data['X_train'][:num_train],
'y_train': data['y_train'][:num_train],
'X_val': data['X_val'],
'y_val': data['y_val'],
}
learning_rate = 1e-2
weight_scale = 5e-2
model = FullyConnectedNet([100, 100, 100, 100],
weight_scale=weight_scale, dtype=np.float64)
solver = Solver(model, small_data,
print_every=10, num_epochs=20, batch_size=25,
update_rule='sgd',
optim_config={
'learning_rate': learning_rate,
}
)
solver.train()
plt.plot(solver.loss_history, 'o')
plt.title('Training loss history')
plt.xlabel('Iteration')
plt.ylabel('Training loss')
plt.show()

(Iteration 1 / 40) loss: 3.571588
(Epoch 0 / 20) train acc: 0.200000; val_acc: 0.112000
(Epoch 1 / 20) train acc: 0.240000; val_acc: 0.105000
(Epoch 2 / 20) train acc: 0.320000; val_acc: 0.139000
(Epoch 3 / 20) train acc: 0.500000; val_acc: 0.147000
(Epoch 4 / 20) train acc: 0.600000; val_acc: 0.140000
(Epoch 5 / 20) train acc: 0.740000; val_acc: 0.165000
(Iteration 11 / 40) loss: 0.819486
(Epoch 6 / 20) train acc: 0.840000; val_acc: 0.139000
(Epoch 7 / 20) train acc: 0.900000; val_acc: 0.141000
(Epoch 8 / 20) train acc: 0.940000; val_acc: 0.137000
(Epoch 9 / 20) train acc: 0.940000; val_acc: 0.136000
(Epoch 10 / 20) train acc: 0.980000; val_acc: 0.136000
(Iteration 21 / 40) loss: 0.220001
(Epoch 11 / 20) train acc: 0.980000; val_acc: 0.133000
(Epoch 12 / 20) train acc: 1.000000; val_acc: 0.134000
(Epoch 13 / 20) train acc: 1.000000; val_acc: 0.140000
(Epoch 14 / 20) train acc: 1.000000; val_acc: 0.146000
(Epoch 15 / 20) train acc: 1.000000; val_acc: 0.142000
(Iteration 31 / 40) loss: 0.071412
(Epoch 16 / 20) train acc: 1.000000; val_acc: 0.144000
(Epoch 17 / 20) train acc: 1.000000; val_acc: 0.142000
(Epoch 18 / 20) train acc: 1.000000; val_acc: 0.144000
(Epoch 19 / 20) train acc: 1.000000; val_acc: 0.142000
(Epoch 20 / 20) train acc: 1.000000; val_acc: 0.141000


## SGD+Momentum¶

モーメント付き確率的勾配降下法は、vanilla SGDよりも急速に深層ネットワークを収束させる傾向がある広く使われているアップデートルールだ。cs231n/optim.pyを開いて、APIを理解するためにファイルトップのドキュメントを読んで、sgd_momentum関数内のSGD+momentum update ruleを実装して、実装をチェックするために下記のコードを実行する。エラーは1e-8以下であらなければならない。

from cs231n.optim import sgd_momentum
N, D = 4, 5
w = np.linspace(-0.4, 0.6, num=N*D).reshape(N, D)
dw = np.linspace(-0.6, 0.4, num=N*D).reshape(N, D)
v = np.linspace(0.6, 0.9, num=N*D).reshape(N, D)
config = {'learning_rate': 1e-3, 'velocity': v}
next_w, _ = sgd_momentum(w, dw, config=config)
expected_next_w = np.asarray([
[ 0.1406,      0.20738947,  0.27417895,  0.34096842,  0.40775789],
[ 0.47454737,  0.54133684,  0.60812632,  0.67491579,  0.74170526],
[ 0.80849474,  0.87528421,  0.94207368,  1.00886316,  1.07565263],
[ 1.14244211,  1.20923158,  1.27602105,  1.34281053,  1.4096    ]])
expected_velocity = np.asarray([
[ 0.5406,      0.55475789,  0.56891579, 0.58307368,  0.59723158],
[ 0.61138947,  0.62554737,  0.63970526,  0.65386316,  0.66802105],
[ 0.68217895,  0.69633684,  0.71049474,  0.72465263,  0.73881053],
[ 0.75296842,  0.76712632,  0.78128421,  0.79544211,  0.8096    ]])
print('next_w error: ', rel_error(next_w, expected_next_w))
print('velocity error: ', rel_error(expected_velocity, config['velocity']))

next_w error:  8.882347033505819e-09
velocity error:  4.269287743278663e-09


それが終わったら、6層SGD/SGD+momentumネットワークの両方を訓練するために下記のコードを実行し、SGD+momentum更新ルールが収束が速いことを確かめる。

num_train = 4000
small_data = {
'X_train': data['X_train'][:num_train],
'y_train': data['y_train'][:num_train],
'X_val': data['X_val'],
'y_val': data['y_val'],
}
solvers = {}
for update_rule in ['sgd', 'sgd_momentum']:
print('running with ', update_rule)
model = FullyConnectedNet([100, 100, 100, 100, 100], weight_scale=5e-2)
solver = Solver(model, small_data,
num_epochs=5, batch_size=100,
update_rule=update_rule,
optim_config={
'learning_rate': 1e-2,
},
verbose=True)
solvers[update_rule] = solver
solver.train()
print()
plt.subplot(3, 1, 1)
plt.title('Training loss')
plt.xlabel('Iteration')
plt.subplot(3, 1, 2)
plt.title('Training accuracy')
plt.xlabel('Epoch')
plt.subplot(3, 1, 3)
plt.title('Validation accuracy')
plt.xlabel('Epoch')
for update_rule, solver in list(solvers.items()):
plt.subplot(3, 1, 1)
plt.plot(solver.loss_history, 'o', label=update_rule)

plt.subplot(3, 1, 2)
plt.plot(solver.train_acc_history, '-o', label=update_rule)

plt.subplot(3, 1, 3)
plt.plot(solver.val_acc_history, '-o', label=update_rule)
for i in [1, 2, 3]:
plt.subplot(3, 1, i)
plt.legend(loc='upper center', ncol=4)
plt.gcf().set_size_inches(20, 30)
plt.show()

running with  sgd
(Iteration 1 / 200) loss: 2.585544
(Epoch 0 / 5) train acc: 0.092000; val_acc: 0.081000
(Iteration 11 / 200) loss: 2.209535
(Iteration 21 / 200) loss: 2.079625
(Iteration 31 / 200) loss: 2.119313
(Epoch 1 / 5) train acc: 0.235000; val_acc: 0.214000
(Iteration 41 / 200) loss: 1.975927
(Iteration 51 / 200) loss: 2.074209
(Iteration 61 / 200) loss: 1.869214
(Iteration 71 / 200) loss: 2.005798
(Epoch 2 / 5) train acc: 0.361000; val_acc: 0.276000
(Iteration 81 / 200) loss: 1.804474
(Iteration 91 / 200) loss: 1.795176
(Iteration 101 / 200) loss: 1.987435
(Iteration 111 / 200) loss: 1.798125
(Epoch 3 / 5) train acc: 0.342000; val_acc: 0.285000
(Iteration 121 / 200) loss: 1.598781
(Iteration 131 / 200) loss: 1.787556
(Iteration 141 / 200) loss: 1.783347
(Iteration 151 / 200) loss: 1.659025
(Epoch 4 / 5) train acc: 0.389000; val_acc: 0.272000
(Iteration 161 / 200) loss: 1.553153
(Iteration 171 / 200) loss: 1.560925
(Iteration 181 / 200) loss: 1.644202
(Iteration 191 / 200) loss: 1.682408
(Epoch 5 / 5) train acc: 0.436000; val_acc: 0.317000

running with  sgd_momentum
(Iteration 1 / 200) loss: 2.710554
(Epoch 0 / 5) train acc: 0.119000; val_acc: 0.109000
(Iteration 11 / 200) loss: 2.213969
(Iteration 21 / 200) loss: 1.989856
(Iteration 31 / 200) loss: 1.877836
(Epoch 1 / 5) train acc: 0.305000; val_acc: 0.297000
(Iteration 41 / 200) loss: 1.917173
(Iteration 51 / 200) loss: 1.786011
(Iteration 61 / 200) loss: 1.844671
(Iteration 71 / 200) loss: 1.708111
(Epoch 2 / 5) train acc: 0.400000; val_acc: 0.333000
(Iteration 81 / 200) loss: 1.651953
(Iteration 91 / 200) loss: 1.825686
(Iteration 101 / 200) loss: 1.618590
(Iteration 111 / 200) loss: 1.571096
(Epoch 3 / 5) train acc: 0.438000; val_acc: 0.342000
(Iteration 121 / 200) loss: 1.584206
(Iteration 131 / 200) loss: 1.564599
(Iteration 141 / 200) loss: 1.507255
(Iteration 151 / 200) loss: 1.530027
(Epoch 4 / 5) train acc: 0.468000; val_acc: 0.342000
(Iteration 161 / 200) loss: 1.441164
(Iteration 171 / 200) loss: 1.520273
(Iteration 181 / 200) loss: 1.556796
(Iteration 191 / 200) loss: 1.448559
(Epoch 5 / 5) train acc: 0.510000; val_acc: 0.358000


/root/.pyenv/versions/py365/lib/python3.6/site-packages/matplotlib/cbook/deprecation.py:107: MatplotlibDeprecationWarning: Adding an axes using the same arguments as a previous axes currently reuses the earlier instance.  In a future version, a new instance will always be created and returned.  Meanwhile, this warning can be suppressed, and the future behavior ensured, by passing a unique label to each axes instance.
warnings.warn(message, mplDeprecation, stacklevel=1)


# Test RMSProp implementation; you should see errors less than 1e-7
from cs231n.optim import rmsprop
N, D = 4, 5
w = np.linspace(-0.4, 0.6, num=N*D).reshape(N, D)
dw = np.linspace(-0.6, 0.4, num=N*D).reshape(N, D)
cache = np.linspace(0.6, 0.9, num=N*D).reshape(N, D)
config = {'learning_rate': 1e-2, 'cache': cache}
next_w, _ = rmsprop(w, dw, config=config)
expected_next_w = np.asarray([
[-0.39223849, -0.34037513, -0.28849239, -0.23659121, -0.18467247],
[-0.132737,   -0.08078555, -0.02881884,  0.02316247,  0.07515774],
[ 0.12716641,  0.17918792,  0.23122175,  0.28326742,  0.33532447],
[ 0.38739248,  0.43947102,  0.49155973,  0.54365823,  0.59576619]])
expected_cache = np.asarray([
[ 0.5976,      0.6126277,   0.6277108,   0.64284931,  0.65804321],
[ 0.67329252,  0.68859723,  0.70395734,  0.71937285,  0.73484377],
[ 0.75037008,  0.7659518,   0.78158892,  0.79728144,  0.81302936],
[ 0.82883269,  0.84469141,  0.86060554,  0.87657507,  0.8926    ]])
print('next_w error: ', rel_error(expected_next_w, next_w))
print('cache error: ', rel_error(expected_cache, config['cache']))

next_w error:  9.524687511038133e-08
cache error:  2.6477955807156126e-09

# Test Adam implementation; you should see errors around 1e-7 or less
N, D = 4, 5
w = np.linspace(-0.4, 0.6, num=N*D).reshape(N, D)
dw = np.linspace(-0.6, 0.4, num=N*D).reshape(N, D)
m = np.linspace(0.6, 0.9, num=N*D).reshape(N, D)
v = np.linspace(0.7, 0.5, num=N*D).reshape(N, D)
config = {'learning_rate': 1e-2, 'm': m, 'v': v, 't': 5}
next_w, _ = adam(w, dw, config=config)
expected_next_w = np.asarray([
[-0.40094747, -0.34836187, -0.29577703, -0.24319299, -0.19060977],
[-0.1380274,  -0.08544591, -0.03286534,  0.01971428,  0.0722929],
[ 0.1248705,   0.17744702,  0.23002243,  0.28259667,  0.33516969],
[ 0.38774145,  0.44031188,  0.49288093,  0.54544852,  0.59801459]])
expected_v = np.asarray([
[ 0.69966,     0.68908382,  0.67851319,  0.66794809,  0.65738853,],
[ 0.64683452,  0.63628604,  0.6257431,   0.61520571,  0.60467385,],
[ 0.59414753,  0.58362676,  0.57311152,  0.56260183,  0.55209767,],
[ 0.54159906,  0.53110598,  0.52061845,  0.51013645,  0.49966,   ]])
expected_m = np.asarray([
[ 0.48,        0.49947368,  0.51894737,  0.53842105,  0.55789474],
[ 0.57736842,  0.59684211,  0.61631579,  0.63578947,  0.65526316],
[ 0.67473684,  0.69421053,  0.71368421,  0.73315789,  0.75263158],
[ 0.77210526,  0.79157895,  0.81105263,  0.83052632,  0.85      ]])
print('next_w error: ', rel_error(expected_next_w, next_w))
print('v error: ', rel_error(expected_v, config['v']))
print('m error: ', rel_error(expected_m, config['m']))

next_w error:  1.1395691798535431e-07
v error:  4.208314038113071e-09
m error:  4.214963193114416e-09


learning_rates = {'rmsprop': 1e-4, 'adam': 1e-3}
print('running with ', update_rule)
model = FullyConnectedNet([100, 100, 100, 100, 100], weight_scale=5e-2)
solver = Solver(model, small_data,
num_epochs=5, batch_size=100,
update_rule=update_rule,
optim_config={
'learning_rate': learning_rates[update_rule]
},
verbose=True)
solvers[update_rule] = solver
solver.train()
print()
plt.subplot(3, 1, 1)
plt.title('Training loss')
plt.xlabel('Iteration')
plt.subplot(3, 1, 2)
plt.title('Training accuracy')
plt.xlabel('Epoch')
plt.subplot(3, 1, 3)
plt.title('Validation accuracy')
plt.xlabel('Epoch')
for update_rule, solver in list(solvers.items()):
plt.subplot(3, 1, 1)
plt.plot(solver.loss_history, 'o', label=update_rule)
plt.subplot(3, 1, 2)
plt.plot(solver.train_acc_history, '-o', label=update_rule)
plt.subplot(3, 1, 3)
plt.plot(solver.val_acc_history, '-o', label=update_rule)
for i in [1, 2, 3]:
plt.subplot(3, 1, i)
plt.legend(loc='upper center', ncol=4)
plt.gcf().set_size_inches(20, 30)
plt.show()

running with  adam
(Iteration 1 / 200) loss: 2.527260
(Epoch 0 / 5) train acc: 0.144000; val_acc: 0.131000
(Iteration 11 / 200) loss: 2.076813
(Iteration 21 / 200) loss: 1.928253
(Iteration 31 / 200) loss: 1.731543
(Epoch 1 / 5) train acc: 0.358000; val_acc: 0.298000
(Iteration 41 / 200) loss: 1.776651
(Iteration 51 / 200) loss: 1.805866
(Iteration 61 / 200) loss: 1.758917
(Iteration 71 / 200) loss: 1.832081
(Epoch 2 / 5) train acc: 0.451000; val_acc: 0.334000
(Iteration 81 / 200) loss: 1.471256
(Iteration 91 / 200) loss: 1.522386
(Iteration 101 / 200) loss: 1.391822
(Iteration 111 / 200) loss: 1.586692
(Epoch 3 / 5) train acc: 0.450000; val_acc: 0.345000
(Iteration 121 / 200) loss: 1.385781
(Iteration 131 / 200) loss: 1.494218
(Iteration 141 / 200) loss: 1.325590
(Iteration 151 / 200) loss: 1.486446
(Epoch 4 / 5) train acc: 0.506000; val_acc: 0.359000
(Iteration 161 / 200) loss: 1.271910
(Iteration 171 / 200) loss: 1.414415
(Iteration 181 / 200) loss: 1.105741
(Iteration 191 / 200) loss: 1.220852
(Epoch 5 / 5) train acc: 0.581000; val_acc: 0.365000

running with  rmsprop
(Iteration 1 / 200) loss: 2.738687
(Epoch 0 / 5) train acc: 0.153000; val_acc: 0.142000
(Iteration 11 / 200) loss: 2.150887
(Iteration 21 / 200) loss: 1.895233
(Iteration 31 / 200) loss: 1.852443
(Epoch 1 / 5) train acc: 0.379000; val_acc: 0.317000
(Iteration 41 / 200) loss: 1.638388
(Iteration 51 / 200) loss: 1.880716
(Iteration 61 / 200) loss: 1.740726
(Iteration 71 / 200) loss: 1.674370
(Epoch 2 / 5) train acc: 0.416000; val_acc: 0.351000
(Iteration 81 / 200) loss: 1.621371
(Iteration 91 / 200) loss: 1.554882
(Iteration 101 / 200) loss: 1.527268
(Iteration 111 / 200) loss: 1.405294
(Epoch 3 / 5) train acc: 0.454000; val_acc: 0.346000
(Iteration 121 / 200) loss: 1.415365
(Iteration 131 / 200) loss: 1.371712
(Iteration 141 / 200) loss: 1.489653
(Iteration 151 / 200) loss: 1.515629
(Epoch 4 / 5) train acc: 0.482000; val_acc: 0.366000
(Iteration 161 / 200) loss: 1.451874
(Iteration 171 / 200) loss: 1.314353
(Iteration 181 / 200) loss: 1.485056
(Iteration 191 / 200) loss: 1.314951
(Epoch 5 / 5) train acc: 0.549000; val_acc: 0.356000


/root/.pyenv/versions/py365/lib/python3.6/site-packages/matplotlib/cbook/deprecation.py:107: MatplotlibDeprecationWarning: Adding an axes using the same arguments as a previous axes currently reuses the earlier instance.  In a future version, a new instance will always be created and returned.  Meanwhile, this warning can be suppressed, and the future behavior ensured, by passing a unique label to each axes instance.
warnings.warn(message, mplDeprecation, stacklevel=1)


## Train a good model!¶

CIFAR-10を使って可能な最良の全結合モデルを訓練し、best_model変数にベストモデルを保存する。検証セットで少なくとも50%の正確性を得ることが求められている。調整次第で、55％を超える精度を得ることもできるはずだが、この宿題ではそこまでは求めないし、それを達成したとしても加点されることはない。宿題2の後半では、CIFAR-10で可能な最良のconvolutional network(畳み込みネットワーク)を訓練することが要求され、全結合ネットよりも畳み込みネットで作業を進めることが推奨される。この部分の宿題に取り掛かる前に、BatchNormalization.ipynbとDropout.ipynbを先に終わらせることを勧める。

################################################################################
# TODO: Train the best FullyConnectedNet that you can on CIFAR-10. You might   #
# batch normalization and dropout useful. Store your best model in the         #
# best_model variable.                                                         #
################################################################################
best_model = FullyConnectedNet([100, 100, 100, 100, 100, 100], weight_scale=1e-5,
use_batchnorm=True, dropout=0.8)
solver = Solver(best_model, data,
num_epochs=20, batch_size=50,
optim_config={'learning_rate': 1e-4},
verbose=False)
solver.train()
##########################
################################################################################
#                              END OF YOUR CODE                                #
################################################################################


## Test you model¶

validation/test setsでベストモデルを実行する。検証セットで50％を超える精度を達成する必要がある。

y_test_pred = np.argmax(best_model.loss(data['X_test']), axis=1)
y_val_pred = np.argmax(best_model.loss(data['X_val']), axis=1)
print('Validation set accuracy: ', (y_val_pred == data['y_val']).mean())
print('Test set accuracy: ', (y_test_pred == data['y_test']).mean())

Validation set accuracy:  0.523
Test set accuracy:  0.534


1. Tijmen Tieleman and Geoffrey Hinton. “Lecture 6.5-rmsprop: Divide the gradient by a running average of its recent magnitude.” COURSERA: Neural Networks for Machine Learning 4 (2012).
2. Diederik Kingma and Jimmy Ba, “Adam: A Method for Stochastic Optimization”, ICLR 2015.
スポンサーリンク

フォローする