前言：
文章以Andrew Ng 的 deeplearning.ai 視頻課程為主線恬吕，記錄Programming Assignments 的實(shí)現(xiàn)過程灌曙。相對(duì)于斯坦福的CS231n課程，Andrew的視頻課程更加簡單易懂，適合深度學(xué)習(xí)的入門者系統(tǒng)學(xué)習(xí)！
本次作業(yè)主要練習(xí)的是最優(yōu)化cost函數(shù)的方法，不同的優(yōu)化方法可以加速學(xué)習(xí)的過程忙迁，可能給最后的識(shí)別準(zhǔn)確率帶來不同的影響。對(duì)于cost函數(shù)的優(yōu)化首先有一個(gè)直觀的感受：
1.1 Gradient Descent：
一個(gè)簡單的優(yōu)化方法叫做梯度下降的方法碎乃，在每次迭代中對(duì)所有樣本執(zhí)行梯度下降姊扔，因此也叫做batch gradient descent
代碼如下：
def update_parameters_with_gd(parameters, grads, learning_rate):
L = len(parameters) // 2
for l in range(L):
parameters["W" + str(l+1)] = parameters["W"+str(l+1)]-learning_rate*grads["dW"+str(l+1)]
parameters["b" + str(l+1)] = parameters["b"+str(l+1)]-learning_rate*grads["db"+str(l+1)]
return parameters
Stochastic Gradient Descent:針對(duì)于每一個(gè)樣本，對(duì)每一個(gè)樣本執(zhí)行梯度下降算法
Mini-Batch Gradient descent 介于SGD和 GD嵌言，每次訓(xùn)練的樣本數(shù)量<m且>1躏率，這樣可以吸取兩種方法的優(yōu)勢(shì)供嚎，達(dá)到好的效果
1.2 Mini-Batch Gradient descent：
我們首先需要構(gòu)建Mini-Batch 去訓(xùn)練模型涉及到兩個(gè)過程shuffle和partition克滴，代碼如下：
def random_mini_batches(X, Y, mini_batch_size = 64, seed = 0):
np.random.seed(seed) ? ? ? ? ??
m = X.shape[1] ? ? ? ? ? ? ? ? ?
mini_batches = []
permutation = list(np.random.permutation(m))
shuffled_X = X[:, permutation]
shuffled_Y = Y[:, permutation].reshape((1,m))
# Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case.
num_complete_minibatches = math.floor(m/mini_batch_size)?
for k in range(0, num_complete_minibatches):
mini_batch_X = shuffled_X[:,k*mini_batch_size:(k+1)*mini_batch_size]
mini_batch_Y = shuffled_Y[:,k*mini_batch_size:(k+1)*mini_batch_size]
mini_batch = (mini_batch_X, mini_batch_Y)
mini_batches.append(mini_batch)
if m % mini_batch_size != 0:
mini_batch_X = shuffled_X[:, num_complete_minibatches*mini_batch_size:m]
mini_batch_Y = shuffled_Y[:, num_complete_minibatches*mini_batch_size:m]
mini_batch = (mini_batch_X, mini_batch_Y)
mini_batches.append(mini_batch)
return mini_batches
1.3 Momentum
def initialize_velocity(parameters):
L = len(parameters) // 2?
v = {}
for l in range(L):
v["dW" + str(l+1)] = np.zeros((parameters["W"+str(l+1)].shape[0],parameters["W"+str(l+1)].shape[1]))
v["db" + str(l+1)] = np.zeros((parameters["b"+str(l+1)].shape[0],parameters["b"+str(l+1)].shape[1]))
return v
def update_parameters_with_momentum(parameters, grads, v, beta, learning_rate):
L = len(parameters) // 2 # number of layers in the neural networks
for l in range(L):
v["dW" + str(l+1)] = beta*v["dW"+str(l+1)]+(1-beta)*grads["dW"+str(l+1)]
v["db" + str(l+1)] = beta*v["db"+str(l+1)]+(1-beta)*grads["db"+str(l+1)]
parameters["W" + str(l+1)] = parameters["W"+str(l+1)]-learning_rate*v["dW"+str(l+1)]
parameters["b" + str(l+1)] = parameters["b"+str(l+1)]-learning_rate*v["db"+str(l+1)]
return parameters, v
1.4 Adam
Adam是目前為止最為廣泛應(yīng)用的優(yōu)化方式逼争，整合了RMSProp和Momentum的優(yōu)點(diǎn)，計(jì)算方式如下：
def initialize_adam(parameters) :
L = len(parameters) // 2?
v = {}
s = {}
for l in range(L):
v["dW" + str(l+1)] = np.zeros((parameters["W"+str(l+1)].shape[0],parameters["W"+str(l+1)].shape[1]))
v["db" + str(l+1)] = np.zeros((parameters["b" + str(l + 1)].shape[0], parameters["b" + str(l + 1)].shape[1]))
s["dW" + str(l+1)] = np.zeros((parameters["W" + str(l + 1)].shape[0], parameters["W" + str(l + 1)].shape[1]))
s["db" + str(l+1)] = np.zeros((parameters["b" + str(l + 1)].shape[0], parameters["b" + str(l + 1)].shape[1]))
return v, s
def update_parameters_with_adam(parameters, grads, v, s, t, learning_rate = 0.01,
beta1 = 0.9, beta2 = 0.999,? epsilon = 1e-8):
L = len(parameters) // 2 ? ? ? ? ? ? ?
s_corrected = {} ? ?
v_corrected = {} ? ? ? ? ? ? ? ??
for l in range(L):
v["dW" + str(l+1)] = beta1*v["dW"+str(l+1)]+(1-beta1)*grads["dW"+str(l+1)]
v["db" + str(l+1)] = beta1*v["db"+str(l+1)]+(1-beta1)*grads["db"+str(l+1)]
v_corrected["dW" + str(l+1)] = v["dW"+str(l+1)]/(1-beta1**t)
v_corrected["db" + str(l+1)] = v["db"+str(l+1)]/(1-beta1**t)
s["dW" + str(l+1)] = beta2*s["dW"+str(l+1)]+(1-beta2)*(grads["dW"+str(l+1)]*grads["dW"+str(l+1)])
s["db" + str(l+1)] = beta2*s["db"+str(l+1)]+(1-beta2)*(grads["db"+str(l+1)]*grads["db"+str(l+1)])
s_corrected["dW" + str(l+1)] = s["dW"+str(l+1)]/(1-beta2**t)
s_corrected["db" + str(l+1)] = s["db"+str(l+1)]/(1-beta2**t)
parameters["W" + str(l+1)] = parameters["W"+str(l+1)]-learning_rate*v_corrected["dW"+str(l+1)]/(np.sqrt(s_corrected["dW"+str(l+1)])+epsilon)
parameters["b" + str(l+1)] = parameters["b"+str(l+1)]-learning_rate*v_corrected["db"+str(l+1)]/(np.sqrt(s_corrected["db"+str(l+1)])+epsilon)
return parameters, v, s
1.5 Model?
首先看一下數(shù)據(jù)集的樣子：
train_X, train_Y = load_dataset()
def model(X, Y, layers_dims, optimizer, learning_rate = 0.0007, mini_batch_size = 64, beta = 0.9,
beta1 = 0.9, beta2 = 0.999,? epsilon = 1e-8, num_epochs = 10000, print_cost = True):
L = len(layers_dims) ? ? ? ? ?
costs = [] ? ? ? ? ? ? ? ? ? ?
t = 0 ? ? ? ? ? ? ? ? ? ? ? ? ?
seed = 10 ? ? ? ? ? ? ? ? ? ? ??
parameters = initialize_parameters(layers_dims)
if optimizer == "gd":
pass?
elif optimizer == "momentum":
v = initialize_velocity(parameters)
elif optimizer == "adam":
v, s = initialize_adam(parameters)
# Optimization loop
for i in range(num_epochs):
seed = seed + 1
minibatches = random_mini_batches(X, Y, mini_batch_size, seed)
for minibatch in minibatches:
(minibatch_X, minibatch_Y) = minibatch
a3, caches = forward_propagation(minibatch_X, parameters)
cost = compute_cost(a3, minibatch_Y)
grads = backward_propagation(minibatch_X, minibatch_Y, caches)
if optimizer == "gd":
parameters = update_parameters_with_gd(parameters, grads, learning_rate)
elif optimizer == "momentum":
parameters, v = update_parameters_with_momentum(parameters, grads, v, beta, learning_rate)
elif optimizer == "adam":
t = t + 1?
parameters, v, s = update_parameters_with_adam(parameters, grads, v, s,
t, learning_rate, beta1, beta2,? epsilon)
if print_cost and i % 1000 == 0:
print ("Cost after epoch %i: %f" %(i, cost))
if print_cost and i % 100 == 0:
costs.append(cost)
plt.plot(costs)
plt.ylabel('cost')
plt.xlabel('epochs (per 100)')
plt.title("Learning rate = " + str(learning_rate))
plt.show()
return parameters
我們看一下 Mini-batch Gradient descent的訓(xùn)練效果：
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer = "gd")
predictions = predict(train_X, train_Y, parameters)
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5,2.5])
axes.set_ylim([-1,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
可以發(fā)現(xiàn)準(zhǔn)確率只有將近80%
下面我們看一下momentum的訓(xùn)練效果：
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, beta = 0.9, optimizer = "momentum")
predictions = predict(train_X, train_Y, parameters)
plt.title("Model with Momentum optimization")
axes = plt.gca()
axes.set_xlim([-1.5,2.5])
axes.set_ylim([-1,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
準(zhǔn)確率基本上和Mini-batch Gradient Descent差不多
最后我們看一下Adam的訓(xùn)練效果：
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer = "adam")
predictions = predict(train_X, train_Y, parameters)
plt.title("Model with Adam optimization")
axes = plt.gca()
axes.set_xlim([-1.5,2.5])
axes.set_ylim([-1,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
我們看到準(zhǔn)確率達(dá)到94%
綜上所述劝赔，我們發(fā)現(xiàn)Momentum通常是有效果的誓焦，但是在較小的學(xué)習(xí)率和簡單的數(shù)據(jù)集上，效果不是很明顯着帽，Adam通常來說效果要由于其他兩種方法杂伟，但是在更多迭代次數(shù)的情況下，通常3種優(yōu)化方法都會(huì)得到一個(gè)好的結(jié)果仍翰，Adam只是收斂的更快赫粥。
最后附上我作業(yè)的得分，表示我程序沒有問題予借，如果覺得我的文章對(duì)您有用越平，請(qǐng)隨意打賞，我將持續(xù)更新Deeplearning.ai的作業(yè)灵迫！
