注意：模擬登陸時病涨，必須保證settings.py里的 COOKIES_ENABLED (Cookies中間件) 處于開啟狀態(tài)
COOKIES_ENABLED = True 或 # COOKIES_ENABLED = False
策略一：直接POST數(shù)據(jù)（比如需要登陸的賬戶信息)
只要是需要提供post數(shù)據(jù)的柬赐，就可以用這種方法行您。下面示例里post的數(shù)據(jù)是賬戶密碼：
# -*- coding: utf-8 -*-
import scrapy
class Renren1Spider(scrapy.Spider):
name = "renren1"
allowed_domains = ["renren.com"]
def start_requests(self):
url = 'http://www.renren.com/PLogin.do'
# FormRequest 是Scrapy發(fā)送POST請求的方法
yield scrapy.FormRequest(
url = url,
formdata = {"email" : "mr_mao_hacker@163.com", "password" : "axxxxxxxe"},
callback = self.parse_page)
def parse_page(self, response):
with open("mao2.html", "w") as filename:
filename.write(response.body)
策略二：標(biāo)準(zhǔn)的模擬登陸步驟
正統(tǒng)模擬登錄方法：
首先發(fā)送登錄頁面的get請求碳蛋，獲取到頁面里的登錄必須的參數(shù)（比如說zhihu登陸界面的 _xsrf）
然后和賬戶密碼一起post到服務(wù)器痛阻，登錄成功
# -*- coding: utf-8 -*-
import scrapy
class Renren2Spider(scrapy.Spider):
name = "renren2"
allowed_domains = ["renren.com"]
start_urls = (
"http://www.renren.com/PLogin.do",
)
# 處理start_urls里的登錄url的響應(yīng)內(nèi)容靖秩，提取登陸需要的參數(shù)（如果需要的話)
def parse(self, response):
# 提取登陸需要的參數(shù)
#_xsrf = response.xpath("http://_xsrf").extract()[0]
# 發(fā)送請求參數(shù)边翼，并調(diào)用指定回調(diào)函數(shù)處理
yield scrapy.FormRequest.from_response(
response,
formdata = {"email" : "mr_mao_hacker@163.com", "password" : "axxxxxxxe"},#, "_xsrf" = _xsrf},
callback = self.parse_page
)
# 獲取登錄成功狀態(tài)猪贪，訪問需要登錄后才能訪問的頁面
def parse_page(self, response):
url = "http://www.renren.com/422167102/profile"
yield scrapy.Request(url, callback = self.parse_newpage)
# 處理響應(yīng)內(nèi)容
def parse_newpage(self, response):
with open("xiao.html", "w") as filename:
filename.write(response.body)
策略三：直接使用保存登陸狀態(tài)的Cookie模擬登陸
如果實(shí)在沒辦法了，可以用這種方法模擬登錄讯私，雖然麻煩一點(diǎn)热押，但是成功率100%
# -*- coding: utf-8 -*-
import scrapy
class RenrenSpider(scrapy.Spider):
name = "renren"
allowed_domains = ["renren.com"]
start_urls = (
'http://www.renren.com/111111',
'http://www.renren.com/222222',
'http://www.renren.com/333333',
)
cookies = {
"anonymid" : "ixrna3fysufnwv",
"_r01_" : "1",
"ap" : "327550029",
"JSESSIONID" : "abciwg61A_RvtaRS3GjOv",
"depovince" : "GW",
"springskin" : "set",
"jebe_key" : "f6fb270b-d06d-42e6-8b53-e67c3156aa7e%7Cc13c37f53bca9e1e7132d4b58ce00fa3%7C1484060607478%7C1%7C1486198628950",
"t" : "691808127750a83d33704a565d8340ae9",
"societyguester" : "691808127750a83d33704a565d8340ae9",
"id" : "327550029",
"xnsid" : "f42b25cf",
"loginfrom" : "syshome"
}
# 可以重寫Spider類的start_requests方法，附帶Cookie值斤寇，發(fā)送POST請求
def start_requests(self):
for url in self.start_urls:
yield scrapy.FormRequest(url, cookies = self.cookies, callback = self.parse_page)
# 處理響應(yīng)內(nèi)容
def parse_page(self, response):
print "===========" + response.url
with open("deng.html", "w") as filename:
filename.write(response.body)