MNIST 卷積神經(jīng)網(wǎng)絡(luò)舰绘。https://github.com/nlintz/TensorFlow-Tutorials/blob/master/05_convolutional_net.py 蹂喻。
TensorFlow搭建卷積神經(jīng)網(wǎng)絡(luò)(CNN)模型，訓(xùn)練MNIST數(shù)據(jù)集捂寿。

構(gòu)建模型口四。

定義輸入數(shù)據(jù)，預(yù)處理數(shù)據(jù)秦陋。讀取數(shù)據(jù)MNIST蔓彩，得到訓(xùn)練集圖片、標(biāo)記矩陣驳概，測試集圖片標(biāo)記矩陣赤嚼。trX、trY顺又、teX探膊、teY 數(shù)據(jù)矩陣表現(xiàn)。trX待榔、teX形狀變?yōu)閇-1,28,28,1]逞壁，-1 不考慮輸入圖片數(shù)量，28x28 圖片長锐锣、寬像素?cái)?shù)腌闯，1 通道(channel)數(shù)量。MNIST 黑白圖片雕憔，通道1姿骏。RGB彩色圖像，通道3斤彼。
初始化權(quán)重分瘦，定義網(wǎng)絡(luò)結(jié)構(gòu)。卷積神經(jīng)網(wǎng)絡(luò)琉苇，3個(gè)卷積層嘲玫、3個(gè)池化層、1個(gè)全連接層并扇、1個(gè)輸出層去团。
定義dropout占位符keep_conv，神經(jīng)元保留比例。生成網(wǎng)絡(luò)模型土陪，得到預(yù)測值昼汗。
定義損失函數(shù)，tf.nn.softmax_cross_entropy_with_logits 比較預(yù)測值鬼雀、真實(shí)值差異顷窒，做均值處理。
定義訓(xùn)練操作(train_op)源哩，RMSProp算法優(yōu)化器tf.train.RMSPropOptimizer蹋肮，學(xué)習(xí)率0.001,衰減值0.9，優(yōu)化損失璧疗。
定義預(yù)測操作(predict_op)坯辩。
會(huì)話啟動(dòng)圖，訓(xùn)練崩侠、評估漆魔。

#!/usr/bin/env python
import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
batch_size = 128 # 訓(xùn)練批次大小
test_size = 256 # 評估批次大小
# 定義初始化權(quán)重函數(shù)
def init_weights(shape):
    return tf.Variable(tf.random_normal(shape, stddev=0.01))
# 定義神經(jīng)網(wǎng)絡(luò)模型函數(shù)
# 入?yún)ⅲ篨 輸入數(shù)據(jù)，w 每層權(quán)重却音，p_keep_conv改抡、p_keep_hidden dropout保留神經(jīng)元比例
def model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden):
    # 第一組卷積層及池化層系瓢，dropout部分神經(jīng)元
    l1a = tf.nn.relu(tf.nn.conv2d(X, w,                       # l1a shape=(?, 28, 28, 32)
                        strides=[1, 1, 1, 1], padding='SAME'))
    l1 = tf.nn.max_pool(l1a, ksize=[1, 2, 2, 1],              # l1 shape=(?, 14, 14, 32)
                        strides=[1, 2, 2, 1], padding='SAME')
    l1 = tf.nn.dropout(l1, p_keep_conv)
    # 第二組卷積層及池化層，dropout部分神經(jīng)元
    l2a = tf.nn.relu(tf.nn.conv2d(l1, w2,                     # l2a shape=(?, 14, 14, 64)
                        strides=[1, 1, 1, 1], padding='SAME'))
    l2 = tf.nn.max_pool(l2a, ksize=[1, 2, 2, 1],              # l2 shape=(?, 7, 7, 64)
                        strides=[1, 2, 2, 1], padding='SAME')
    l2 = tf.nn.dropout(l2, p_keep_conv)
    # 第三組卷積層及池化層欠拾，dropout部分神經(jīng)元
    l3a = tf.nn.relu(tf.nn.conv2d(l2, w3,                     # l3a shape=(?, 7, 7, 128)
                        strides=[1, 1, 1, 1], padding='SAME'))
    l3 = tf.nn.max_pool(l3a, ksize=[1, 2, 2, 1],              # l3 shape=(?, 4, 4, 128)
                        strides=[1, 2, 2, 1], padding='SAME')
    l3 = tf.reshape(l3, [-1, w4.get_shape().as_list()[0]])    # reshape to (?, 2048)
    l3 = tf.nn.dropout(l3, p_keep_conv)
    # 全連接層，dropout部分神經(jīng)元
    l4 = tf.nn.relu(tf.matmul(l3, w4))
    l4 = tf.nn.dropout(l4, p_keep_hidden)
    # 輸出層
    pyx = tf.matmul(l4, w_o)
    return pyx # 返回預(yù)測值
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
# 數(shù)據(jù)預(yù)處理
trX = trX.reshape(-1, 28, 28, 1)  # 28x28x1 input img
teX = teX.reshape(-1, 28, 28, 1)  # 28x28x1 input img
X = tf.placeholder("float", [None, 28, 28, 1])
Y = tf.placeholder("float", [None, 10])
# 卷積核大小 3x3
# patch大小3x3骗绕，輸入維度1，輸出維度32
w = init_weights([3, 3, 1, 32])       # 3x3x1 conv, 32 outputs
# patch大小3x3荆忍，輸入維度32，輸出維度64
w2 = init_weights([3, 3, 32, 64])     # 3x3x32 conv, 64 outputs
# patch大小3x3撤缴，輸入維度64刹枉，輸出維度128
w3 = init_weights([3, 3, 64, 128])    # 3x3x32 conv, 128 outputs
# 全連接層，輸入維度128*4*4 上層輸數(shù)據(jù)三維轉(zhuǎn)一維微宝，輸出維度625
w4 = init_weights([128 * 4 * 4, 625]) # FC 128 * 4 * 4 inputs, 625 outputs
# 輸出層凉袱，輸入維度625侦铜，輸出維度10 代表10類(labels)
w_o = init_weights([625, 10])         # FC 625 inputs, 10 outputs (labels)
# 定義dropout占位符
p_keep_conv = tf.placeholder("float")
p_keep_hidden = tf.placeholder("float")
py_x = model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden) # 得到預(yù)測值
# 定義損失函數(shù)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=Y))
# 定義訓(xùn)練操作
train_op = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cost)
# 定義預(yù)測操作
predict_op = tf.argmax(py_x, 1)
# Launch the graph in a session
#會(huì)話啟動(dòng)圖
with tf.Session() as sess:
    # you need to initialize all variables
    tf.global_variables_initializer().run()
    for i in range(100):
        # 訓(xùn)練模型
        training_batch = zip(range(0, len(trX), batch_size),
                             range(batch_size, len(trX)+1, batch_size))
        for start, end in training_batch:
            sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end],
                                          p_keep_conv: 0.8, p_keep_hidden: 0.5})
        # 評估模型
        test_indices = np.arange(len(teX)) # Get A Test Batch
        np.random.shuffle(test_indices)
        test_indices = test_indices[0:test_size]
        print(i, np.mean(np.argmax(teY[test_indices], axis=1) ==
                         sess.run(predict_op, feed_dict={X: teX[test_indices],
                                                         p_keep_conv: 1.0,
                                                         p_keep_hidden: 1.0})))

MNIST 循環(huán)神經(jīng)網(wǎng)絡(luò)。 https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/recurrent_network.py 棺耍。

RNN 自然語言處理領(lǐng)域成功應(yīng)用种樱，機(jī)器翻譯嫩挤、語音識別、圖像描述生成(圖像特征生成描述)岂昭、語言模型與文本生成(生成模型預(yù)測下一單詞概率)约啊。Alex Graves《Supervised Sequence Labelling with Recurrent Neural Networks》 http://www.cs.toronto.edu/~graves/preprint.pdf 。

構(gòu)建模型记盒。設(shè)置訓(xùn)練超參數(shù)外傅，設(shè)置學(xué)習(xí)率、訓(xùn)練次數(shù)彬碱、每輪訓(xùn)練數(shù)據(jù)大小奥洼。
RNN分類圖片灵奖，每張圖片行，像素序列(sequence)瓷患。MNIST圖片大小28x28擅编，28個(gè)元素序列 X 28行箫踩，每步輸入序列長度28谭贪，輸入步數(shù)28步俭识。
定義輸入數(shù)據(jù)、權(quán)重缚态。
定義RNN模型堤瘤。
定義損失函數(shù)、優(yōu)化器(AdamOptimizer)桥帆。
定義模型預(yù)測結(jié)果师郑、準(zhǔn)確率計(jì)算方法张遭。
會(huì)話啟動(dòng)圖地梨，開始訓(xùn)練，每20次輸出1次準(zhǔn)確率大小洁闰。

from __future__ import print_function
import tensorflow as tf
from tensorflow.contrib import rnn
# Import MNIST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
# Training Parameters
# 設(shè)置訓(xùn)練超參數(shù)
learning_rate = 0.001
training_steps = 10000
batch_size = 128
display_step = 200
# Network Parameters
# 神經(jīng)網(wǎng)絡(luò)參數(shù)
num_input = 28 # MNIST data input (img shape: 28*28) 輸入層
timesteps = 28 # timesteps 28 長度
num_hidden = 128 # hidden layer num of features 隱藏層神經(jīng)元數(shù)
num_classes = 10 # MNIST total classes (0-9 digits) 輸出數(shù)量万细，分類類別 0~9
# tf Graph input
# 輸入數(shù)據(jù)占位符
X = tf.placeholder("float", [None, timesteps, num_input])
Y = tf.placeholder("float", [None, num_classes])
# Define weights
# 定義權(quán)重
weights = {
    'out': tf.Variable(tf.random_normal([num_hidden, num_classes]))
}
biases = {
    'out': tf.Variable(tf.random_normal([num_classes]))
}
# 定義RNN模型
def RNN(x, weights, biases):
    # Unstack to get a list of 'timesteps' tensors of shape (batch_size, n_input)
    # 輸入x轉(zhuǎn)換成(128 batch * 28 steps, 28 inputs)
    x = tf.unstack(x, timesteps, 1)
    # Define a lstm cell with tensorflow
    # 基本LSTM循環(huán)網(wǎng)絡(luò)單元 BasicLSTMCell
    lstm_cell = rnn.BasicLSTMCell(num_hidden, forget_bias=1.0)
    # Get lstm cell output
    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights['out']) + biases['out']
logits = RNN(X, weights, biases)
prediction = tf.nn.softmax(logits)
# Define loss and optimizer
# 定義損失函數(shù)
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
    logits=logits, labels=Y))
# 定義優(yōu)化器
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)
# Evaluate model (with test logits, for dropout to be disabled)
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
# Start training
with tf.Session() as sess:
    # Run the initializer
    sess.run(init)
    for step in range(1, training_steps+1):
        batch_x, batch_y = mnist.train.next_batch(batch_size)
        # Reshape data to get 28 seq of 28 elements
        batch_x = batch_x.reshape((batch_size, timesteps, num_input))
        # Run optimization op (backprop)
        sess.run(train_op, feed_dict={X: batch_x, Y: batch_y})
        if step % display_step == 0 or step == 1:
            # Calculate batch loss and accuracy
            loss, acc = sess.run([loss_op, accuracy], feed_dict={X: batch_x,
                                                                 Y: batch_y})
            print("Step " + str(step) + ", Minibatch Loss= " + \
                  "{:.4f}".format(loss) + ", Training Accuracy= " + \
                  "{:.3f}".format(acc))
    print("Optimization Finished!")
    # Calculate accuracy for 128 mnist test images
    test_len = 128
    test_data = mnist.test.images[:test_len].reshape((-1, timesteps, num_input))
    test_label = mnist.test.labels[:test_len]
    print("Testing Accuracy:", \
        sess.run(accuracy, feed_dict={X: test_data, Y: test_label}))

MNIST 無監(jiān)督學(xué)習(xí)腰素。自編碼器(autoencoder)雪营。

自編碼網(wǎng)絡(luò)。UFLDL http://ufldl.stanford.edu/wiki/index.php/Autoencoders_and_Sparsity 洋访。
監(jiān)督學(xué)習(xí)數(shù)據(jù)有標(biāo)記姻政。
自編碼網(wǎng)絡(luò)，輸入樣本壓縮到隱藏層扶歪，解壓摄闸，輸出端重建樣本善镰。最終輸出層神經(jīng)元數(shù)量等于輸入層神經(jīng)元數(shù)據(jù)量炫欺。壓縮熏兄，輸入數(shù)據(jù)(圖像、文本摩桶、聲音)存在不同程度冗余信息硝清，自動(dòng)編碼網(wǎng)絡(luò)學(xué)習(xí)去掉冗余信息，有用特征輸入到隱藏層芦拿。找到可以代表源數(shù)據(jù)的主要成分。激活函數(shù)不使用sigmoid等非線性函數(shù)蔗崎，用線性函數(shù)缓苛，就是PCA模型。
主成分分析(principal components analysis, PCA)笔刹，分析钢属、簡化數(shù)據(jù)集技術(shù)淆党。減少數(shù)據(jù)集維數(shù)讶凉，保持?jǐn)?shù)據(jù)集方差貢獻(xiàn)最大特征山孔。保留低階主成分，忽略高階主成分褐望。最常用線性降維方法串前。
壓縮過程荡碾，限制隱藏神經(jīng)元數(shù)量，學(xué)習(xí)有意義特征劳殖。希望神經(jīng)元大部分時(shí)間被抑制拨脉。神經(jīng)元輸出接近1為被激活，接近0為被抑制矛缨。部分神經(jīng)元處于被抑制狀態(tài)劳景，稀疏性限制碉就。
多個(gè)隱藏層，輸入數(shù)據(jù)圖像筋量，第一層學(xué)習(xí)識別邊碉熄，第二層學(xué)習(xí)組合邊锈津，構(gòu)成輪廓、角性誉，更高層學(xué)習(xí)組合更有意義特征。

TensorFlow自編碼網(wǎng)絡(luò)實(shí)現(xiàn)纫雁。 https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/autoencoder.py 倾哺。

構(gòu)建模型羞海。設(shè)置超參數(shù)，學(xué)習(xí)率菜循、訓(xùn)練輪數(shù)(epoch)申尤、每次訓(xùn)練數(shù)據(jù)多少昧穿、每隔多少輪顯示一次訓(xùn)練結(jié)果橙喘。
定義輸入數(shù)據(jù)，無監(jiān)督學(xué)習(xí)只需要圖片數(shù)據(jù)饰潜，不需要標(biāo)記數(shù)據(jù)和簸。
初始化權(quán)重锁保，定義網(wǎng)絡(luò)結(jié)構(gòu)。2個(gè)隱藏層吴菠，第一個(gè)隱藏層神經(jīng)元256個(gè)浩村，第二個(gè)隱藏層神經(jīng)元128個(gè)心墅。包括壓縮重挑、解壓過程棠涮。
構(gòu)建損失函數(shù)严肪、優(yōu)化器。損失函數(shù)“最小二乘法”篇梭，原始數(shù)據(jù)集和輸出數(shù)據(jù)集平方差取均值運(yùn)算酝枢。優(yōu)化器用RMSPropOptimizer帘睦。
訓(xùn)練數(shù)據(jù)、評估模型诡延。對測試集應(yīng)用訓(xùn)練好的自動(dòng)編碼網(wǎng)絡(luò)古胆。比較測試集原始圖片和自動(dòng)編碼網(wǎng)絡(luò)重建結(jié)果逸绎。

from __future__ import division, print_function, absolute_import
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
# Import MNIST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
# Training Parameters
# 設(shè)置訓(xùn)練超參數(shù)
learning_rate = 0.01 # 學(xué)習(xí)率
num_steps = 30000 # 訓(xùn)練輪數(shù)
batch_size = 256 # 每次訓(xùn)練數(shù)據(jù)多少
display_step = 1000 # 每隔多少輪顯示訓(xùn)練結(jié)果
examples_to_show = 10 # 測試集選10張圖片驗(yàn)證自動(dòng)編碼器結(jié)果
# Network Parameters
# 網(wǎng)絡(luò)參數(shù)
# 第一個(gè)隱藏層神經(jīng)元個(gè)數(shù)棺牧，特征值個(gè)數(shù)
num_hidden_1 = 256 # 1st layer num features
# 第二個(gè)隱藏層神經(jīng)元個(gè)數(shù)，特征值個(gè)數(shù)
num_hidden_2 = 128 # 2nd layer num features (the latent dim)
# 輸入數(shù)據(jù)特征值個(gè)數(shù) 28x28=784
num_input = 784 # MNIST data input (img shape: 28*28)
# tf Graph input (only pictures)
# 定義輸入數(shù)據(jù)曲秉，只需要圖片承二，不要需要標(biāo)記
X = tf.placeholder("float", [None, num_input])
# 初始化每層權(quán)重和偏置
weights = {
    'encoder_h1': tf.Variable(tf.random_normal([num_input, num_hidden_1])),
    'encoder_h2': tf.Variable(tf.random_normal([num_hidden_1, num_hidden_2])),
    'decoder_h1': tf.Variable(tf.random_normal([num_hidden_2, num_hidden_1])),
    'decoder_h2': tf.Variable(tf.random_normal([num_hidden_1, num_input])),
}
biases = {
    'encoder_b1': tf.Variable(tf.random_normal([num_hidden_1])),
    'encoder_b2': tf.Variable(tf.random_normal([num_hidden_2])),
    'decoder_b1': tf.Variable(tf.random_normal([num_hidden_1])),
    'decoder_b2': tf.Variable(tf.random_normal([num_input])),
}
# Building the encoder
# 定義壓縮函數(shù)
def encoder(x):
    # Encoder Hidden layer with sigmoid activation #1
    layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['encoder_h1']),
                                   biases['encoder_b1']))
    # Encoder Hidden layer with sigmoid activation #2
    layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['encoder_h2']),
                                   biases['encoder_b2']))
    return layer_2
# Building the decoder
# 定義解壓函數(shù)
def decoder(x):
    # Decoder Hidden layer with sigmoid activation #1
    layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['decoder_h1']),
                                   biases['decoder_b1']))
    # Decoder Hidden layer with sigmoid activation #2
    layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['decoder_h2']),
                                   biases['decoder_b2']))
    return layer_2
# Construct model
# 構(gòu)建模型
encoder_op = encoder(X)
decoder_op = decoder(encoder_op)
# Prediction
# 得出預(yù)測值
y_pred = decoder_op
# Targets (Labels) are the input data.
# 得出真實(shí)值亥鸠，即輸入值
y_true = X
# Define loss and optimizer, minimize the squared error
# 定義損失函數(shù)、優(yōu)化器
loss = tf.reduce_mean(tf.pow(y_true - y_pred, 2))
optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
# Start Training
# Start a new TF session
with tf.Session() as sess:
    # Run the initializer
    sess.run(init)
    # Training
    # 開始訓(xùn)練
    for i in range(1, num_steps+1):
        # Prepare Data
        # Get the next batch of MNIST data (only images are needed, not labels)
        batch_x, _ = mnist.train.next_batch(batch_size)
        # Run optimization op (backprop) and cost op (to get loss value)
        _, l = sess.run([optimizer, loss], feed_dict={X: batch_x})
        # Display logs per step
        # 每一輪，打印出一次損失值
        if i % display_step == 0 or i == 1:
            print('Step %i: Minibatch Loss: %f' % (i, l))
    # Testing
    # Encode and decode images from test set and visualize their reconstruction.
    n = 4
    canvas_orig = np.empty((28 * n, 28 * n))
    canvas_recon = np.empty((28 * n, 28 * n))
    for i in range(n):
        # MNIST test set
        batch_x, _ = mnist.test.next_batch(n)
        # Encode and decode the digit image
        g = sess.run(decoder_op, feed_dict={X: batch_x})
        # Display original images
        for j in range(n):
            # Draw the original digits
            canvas_orig[i * 28:(i + 1) * 28, j * 28:(j + 1) * 28] = \
                batch_x[j].reshape([28, 28])
        # Display reconstructed images
        for j in range(n):
            # Draw the reconstructed digits
            canvas_recon[i * 28:(i + 1) * 28, j * 28:(j + 1) * 28] = \
                g[j].reshape([28, 28])
    print("Original Images")
    plt.figure(figsize=(n, n))
    plt.imshow(canvas_orig, origin="upper", cmap="gray")
    plt.show()
    print("Reconstructed Images")
    plt.figure(figsize=(n, n))
    plt.imshow(canvas_recon, origin="upper", cmap="gray")
    plt.show()

參考資料：
《TensorFlow技術(shù)解析與實(shí)戰(zhàn)》

歡迎推薦上海機(jī)器學(xué)習(xí)工作機(jī)會(huì)鸵荠，我的微信：qingxingfengzi