Learning PyTorch
Autograd
autograd.Variable
<u>新版本的pytorch Tensor和Variab合并了</u>
Variable主要包含三個屬性
- data: 保存Variable所含的Tensor
- grad: 保存data對應(yīng)的梯度，grad也是一個Variable，不是Tensor
- grad_fn: 指向一個Function對象，用來反向傳播計算輸入的梯度
x = Variable(t.ones(3), requires_grad=True) # Tensor初始化的時候不能用requires_grad初始化署照，需要單獨x.requires_grad=True改變該屬性
y = t.sum(x)
y.backward() # 反向傳播
x.grad # 得到y(tǒng)對x的導(dǎo)數(shù)
# !! x.grad在反向傳播過程中是累加蒲祈，每次運行反向傳播簇秒，梯度都會累加之前的梯度焊傅，所以反向傳播需把之前梯度清零
x.grad.data.zero_() # 帶_操作是inplace操作
保存和加載模型
保存和提取主要使用torch.save和torch.load方法實現(xiàn)保存和提取
test_data = torch.FloatTensor(2,3)
torch.save(test_data, "test_data.pkl")
test_data_load = torch.load("test_data.pkl")
只保存和加載模型參數(shù)
net = torch.nn.Sequential(
torch.nn.Linear(1,10),
torch.nn.ReLU(),
torch.nn.Linear(10,1)
)
torch.save(net.state_dict(), "net_params.pkl") # 保存模型參數(shù)
net.load_state_dict(torch.load('net_params.pkl')) # 將保存的參數(shù)復(fù)制到net策幼，這種方式會提取所有參數(shù)肺缕，放到現(xiàn)有的網(wǎng)路
保存和加載整個模型
torch.save(net, "net_total.pkl")
net = torch.load('net_total.pkl') # 提取整個網(wǎng)絡(luò)左医，網(wǎng)絡(luò)大的時候費時
<u>同時保存epoch和模型</u>
torch.save({'epoch':epoch, 'state_dict':net.state_dict()}, 'checkpoint.tar')
checkpoint = torch.load('checkpoint.tar')
epoch = checkpoint['epoch']
net.load_state_dict(checkpoint['state_dict'])
nn
nn.Conv2d
nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias)
- padding=0表示四周不進行零填充， padding=1表示四周進行1個像素點的零填充
- bias是一個布爾值同木，默認bias=True浮梢，表示使用偏置
- groups表示輸出數(shù)據(jù)體深度上和輸入數(shù)據(jù)體深度上的聯(lián)系， 默認groups=1泉手，也就是所有的輸入和輸出都是相關(guān)聯(lián)的黔寇，如果groups=2，這表示輸入的深度被分割成兩份斩萌，輸入的深度被分成兩份缝裤，輸出的深度也被分成兩份，它們之間分別對應(yīng)起來颊郎，要求輸入輸出的深度都能被groups整除
- dilation表示卷積對輸入數(shù)據(jù)體的空間間隔（空洞卷積）
nn.MaxPool2d/AvgPool2d
nn.MaxPool2d(kernel_size, stride, padding, dilation, return_indices, ceil_mode)
nn.AvgPool2d(kernel_size, stride, padding, dilation, return_indices, ceil_mode憋飞， count_include_pad=True)
- stride 默認和kernel_size一樣
- return_indices表示是否返回最大值所在處的下標，默認為False
- ceil_mode ceil模式就是會把不足square_size的邊給保留下來姆吭，單獨另算榛做，或者也可以理解為在原來的數(shù)據(jù)上補充了值為-NAN的邊。而floor模式則是直接把不足square_size的邊給舍棄了。
- count_include_pad表示計算均值時是否包含零填充
nn.ZeroPad2d
在輸入的數(shù)據(jù)周圍做zero-padding检眯， 參數(shù)可以是一個int或者一個tuple厘擂，tuple的順序是（左，右锰瘸，上刽严，下）
>>> m = nn.ZeroPad2d(2)
>>> input = torch.randn(1, 1, 3, 3)
>>> input
tensor([[[[-0.1678, -0.4418, 1.9466],
[ 0.9604, -0.4219, -0.5241],
[-0.9162, -0.5436, -0.6446]]]])
>>> m(input)
tensor([[[[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, -0.1678, -0.4418, 1.9466, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.9604, -0.4219, -0.5241, 0.0000, 0.0000],
[ 0.0000, 0.0000, -0.9162, -0.5436, -0.6446, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]]]])
>>> # using different paddings for different sides
>>> m = nn.ZeroPad2d((1, 1, 2, 0))
>>> m(input)
tensor([[[[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, -0.1678, -0.4418, 1.9466, 0.0000],
[ 0.0000, 0.9604, -0.4219, -0.5241, 0.0000],
[ 0.0000, -0.9162, -0.5436, -0.6446, 0.0000]]]])
nn.CrossEntropyLoss
輸出比輸入少一個維度
x = torch.Tensor([[0.3, 0.7]])
y = torch.LongTensor([1])
loss = torch.nn.CrossEntropyLoss()
loss(x, y)
>>> tensor(0.5130)
nn.LogSoftmax
輸入輸出維度相同
x = torch.Tensor([0.3, 0.7])
sm = torch.nn.LogSoftmax()
sm(x)
>>> tensor([-0.9130, -0.5130])
nn.NLLLoss
x = torch.Tensor([[0.3, 0.7]])
y = torch.LongTensor([1])
loss = torch.nn.NLLLoss()
loss(x,y)
>>> tensor(-0.7000)
nn.BCELoss
x = torch.Tensor([0.3, 0.7])
y = torch.Tensor([0,1])
loss = torch.nn.BCELoss(reduce=False)
loss(x,y)
>>> tensor([0.3567, 0.3567]) # 0*ln(0.3) + 1*ln(1-0.3) + 1*ln(0.7)
nn.Module
detach
截斷反向傳播的梯度流，返回一個不再計算梯度的新Variable避凝。在GAN中會用到舞萄。
nn.Sequential
model.add_module(name, module)
model.add_module('conv{0]'.format(conv_id), nn.Conv2d(in_channel, out_channel,kernel_size, stride, pad))
utils
utils.data
制作自己的數(shù)據(jù)集
x = torch.linspace(1,10,10)
y = torch.linspace(10,1,10)
torch_dataset = data.TensorDataset(data_tensor=x, target_tensor=y)
loader = data.DataLoader(
dataset=torch_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2 # 多線程來讀取數(shù)據(jù)
)
Others
numel
返回矩陣元素的個數(shù)，即把所有維數(shù)相乘
x = torch.randn(2,3,4)
torch.numel(x) # output:24 = 2*3*4
add
x.add(y) # 返回新的tensor
x.add_(y) # 改變x
result = t.Tensor(5,3)
.add(x,y, out=result) # 輸入到result
numpy
a = np.ones(3)
b = t.from_numpy(a)
b.add_(1)
# a 的數(shù)值也跟著+1管削， Tensor和numpy對象共享內(nèi)存倒脓，其中一個變了，另外一個也跟著變了
randperm
產(chǎn)生長度為10的隨機排列區(qū)間在[0,9]
t.randperm(10)
unsqueeze
b.unsqueeze(1) # 在第1維（下標從0開始）增加1個維度
LeNet
import torch.nn as nn
import torch.nn.functional as F
class Net(nn.Module):
def __init__(self):
super(Net,self).__init__()
# 1：輸入圖片為單通道 6：輸出通道 5：卷積核5*5
self.conv1 = nn.Conv2d(1,6,5)
self.conv2 = nn.Conv2d(6,16,5)
self.fc1 = nn.Linear(16*5*5, 120)
self.fc2 = nn.Linear(120, 84)
self.fc3 = nn.Linear(84,10)
def forward(self, x):
x = F.max_pool2d(F.relu(self.conv1(x), (2,2)))
x = F.max_pool2d(F.relu(self.conv2(x), 2))
x = x.view(x.size()[0], -1) # 拉直
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.fc3(x)
return x
net = Net()
print(net)
for name, param in net.named_parameters():
print(name, ":", parameters.size())
input = Variable(t.randn(1,1,32,32))
out = net(input)
out.size()
net.zero_grad() # 所有參數(shù)梯度清零
out.backward()
只要在nn.Module的子類中定義了forward含思，backward函數(shù)會自動實現(xiàn)崎弃。在forward函數(shù)中可以使用任何Variable支持的函數(shù)。
網(wǎng)絡(luò)的可學(xué)習(xí)參數(shù)通過net.parameters()返回茸俭，net.named_parameters可同時返回可學(xué)習(xí)的參數(shù)及名稱
SPP_Net
image.png
計算公式
class SPP(nn.Module):
def __init__(self, num_levels, pool_type='max_pool'):
super(SPP, self).__init__()
self.num_levels = num_levels
self.pool_type = pool_type
def forward(self, x):
num, c, h, w = x.size()
for i in range(self.num_levels):
level = i+1
# 當層數(shù)spp層數(shù)多的時候可能出現(xiàn)padding大于kernel_size/2的情況吊履，此時torch的max_pooling2d會報錯，所以使用
kernel_size = (math.ceil(h / level), math.ceil(w / level))
stride = (math.floor(h / level), math.floor(w / level))
padding = (math.floor((kernel_size[0]*level-h+1)/2), math.floor((kernel_size[1]*level-w+1)/2))
# update input data with padding
zero_pad = torch.nn.ZeroPad2d((padding[1],padding[1],padding[0],padding[0])) #left,right,top,botton
x_new = zero_pad(x)
# update kernel and stride
h_new = 2*padding[0] + h
w_new = 2*padding[1] + w
kernel_size = (math.ceil(h_new / level), math.ceil(w_new / level))
stride = (math.floor(h_new / level), math.floor(w_new / level))
#kernel_size = (math.ceil(h/level), math.ceil(w/level))
#stride = (math.ceil(h/level), math.ceil(w/level))
#padding = (math.floor((kernel_size[0]*level-h+1)/2), math.floor((kernel_size[1]*level-w+1)/2))
# 如果不自己調(diào)用ZeroPad2d调鬓，還需在調(diào)用max_pool2d時傳入padding=padding
if self.pool_type == 'max_pool':
tensor = F.max_pool2d(x, kernel_size=kernel_size, stride=stride).view(num, -1)
else:
tensor = F.avg_pool2d(x, kernel_size=kernel_size, stride=stride).view(num, -1)
if(i == 0):
x_flatten = tensor.view(num, -1)
else:
x_flatten = torch.cat((x_flatten, tensor), 1)
return x_flatten
spp = SPP(5) # 1*1 + 2*2 + 3*3 + 4*4 + 5*5 = 55
#print(list(spp.parameters()))
for name, parameters in list(spp.named_parameters()):
print(name, ":", parameters.size())
input = Variable(torch.randn(1,1,7,9))
out = spp(input)
print(out.size()) # (1,55) output size isn't determined by input shape
