之前學(xué)習(xí)了模擬登錄新浪微博，沒(méi)有用到框架，今天晚上嘗試用scrapy重新寫(xiě)一遍模擬登錄善榛。
存在問(wèn)題：
1缚柏、scrapy 好像不擅長(zhǎng)模擬登錄苹熏，寫(xiě)出來(lái)的代碼結(jié)構(gòu)還不如requests庫(kù)寫(xiě)的清楚，當(dāng)然跟我個(gè)人水平有關(guān)币喧，scrapy不太會(huì)轨域。
2、FormRequest的formdata中不能有int型的字段杀餐，不然要報(bào)錯(cuò)干发，網(wǎng)上解決方法要修改scrapy源碼，對(duì)我來(lái)說(shuō)難度太大史翘，放棄了枉长，直接強(qiáng)制轉(zhuǎn)換為str類型，所幸能成功訪問(wèn)琼讽。
3必峰、scrapy 函數(shù)之間的相互調(diào)用很麻煩，我直接順序callback一個(gè)個(gè)函數(shù)钻蹬，有些沒(méi)有關(guān)系的函數(shù)也強(qiáng)制順序訪問(wèn)了吼蚁，而不是在需要的時(shí)候訪問(wèn)，導(dǎo)致程序的結(jié)構(gòu)不清问欠。
4肝匆、勉強(qiáng)能用，后續(xù)改進(jìn)空間大顺献。
5旗国、爬到的首頁(yè)數(shù)據(jù)沒(méi)有什么內(nèi)容，網(wǎng)站用了好多js滚澜，暫時(shí)不知到如何后續(xù)爬區(qū)粗仓。
6、拋磚引玉设捐，希望哪個(gè)大神能看到我的文章借浊，指點(diǎn)我一下如何修改能使爬蟲(chóng)結(jié)構(gòu)更清楚。
from scrapy.http import Request,FormRequest
from scrapy.spiders import CrawlSpider
from urllib.parse import quote_plus
from PIL import Image
import base64
import time
import rsa
import binascii
import requests
import re
class weibo_spider(CrawlSpider):
name = 'weibo'
session = requests.session()
user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
headers = {
'User-Agent': user_agent
}
def start_requests(self):
start_url = 'http://weibo.com/login.php'
return[Request(start_url,meta={'cookiejar':1},callback=self.get_su)]
def get_su(self,response):
username = input('請(qǐng)輸入用戶名：')
username_quote = quote_plus(username)
uername_base64 = base64.b64encode(username_quote.encode('utf-8'))
uername_base64 = uername_base64.decode('utf-8')
print(uername_base64)
#print(response.url)
meta = {'cookiejar':response.meta['cookiejar'],
'uername_base64':uername_base64}
pre_url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su="\
+ uername_base64 + "&rsakt=mod&checkpin=1&client=ssologin.js(v1.4.18)&_=" + str(int(time.time() * 1000))
return[Request(pre_url,meta=meta,callback=self.server_data)]
def server_data(self,response):
#eval的作用是把list,tuple,dict和string相互轉(zhuǎn)化
server_data = eval(response.body.decode('utf-8').replace("sinaSSOController.preloginCallBack",''))
print(server_data)
meta =response.meta
meta['server_data'] = server_data
return[Request('https://baidu.com',meta=meta,callback=self.get_password)]
def get_password(self,response):
password = input('請(qǐng)輸入密碼:')
servertime = response.meta['server_data']['servertime']
nonce = response.meta['server_data']['nonce']
pubkey = response.meta['server_data']['pubkey']
rsapublickey = int(pubkey,16) #轉(zhuǎn)話為10進(jìn)制
key = rsa.PublicKey(rsapublickey,65537)
message = str(servertime) + '\t' + str(nonce) + '\n' + str(password)
message = message.encode("utf-8")
passwd = rsa.encrypt(message, key)
passwd = binascii.b2a_hex(passwd)
print(passwd)
meta =response.meta
meta['passwd'] = passwd
return[Request('http://www.sina.com.cn/',meta=meta,callback=self.login)]
def login(self,response):
su = response.meta['uername_base64']
servertime = response.meta['server_data']['servertime']
nonce = response.meta['server_data']['nonce']
rsakv = response.meta['server_data']['rsakv']
password_secret = response.meta['passwd']
postdata = {
'entry':'weibo',
'gateway':'1',
'from':'',
'savestate':'7',
'useticket':'1',
'pagerefer':"http://login.sina.com.cn/sso/logout.php?entry=miniblog&r=http%3A%2F%2Fweibo.com%2Flogout.php%3Fbackurl",
'wsseretry':'servertime_error',
'vsnf':'1',
'su':su,
'service':'miniblog',
'servertime':str(servertime),
'nonce':nonce,
'pwencode':'rsa2',
'rsakv':rsakv,
'sp':password_secret,
'sr':'1536*864',
'encoding':'UTF-8',
'prelt':'105',
'url':'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
'returntype':'META'
}
login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.18)'
print(postdata)
if response.meta['server_data']['showpin'] == 0:
return[FormRequest(login_url,meta={'cookiejar':response.meta['cookiejar']},formdata=postdata,callback=self.get_loop_url,dont_filter = True)]
"""
else: #獲得驗(yàn)證嗎萝招，手動(dòng)輸入
pcid = response.meta['server_data']['picd']
cha_url = "http://login.sina.com.cn/cgi/pin.php?r="
cha_url = cha_url + str(int(random.random() * 100000000)) + "&s=0&p="
cha_url = cha_url + pcid
cha_page = session.get(cha_url,cookies=response.meta['cookiejar'],headers=headers)
with open("cha.jpg", 'wb') as f:
f.write(cha_page.content)
f.close()
try:
im = Image.open("cha.jpg")
im.show()
im.close()
except:
print(u"請(qǐng)到當(dāng)前目錄下蚂斤，找到驗(yàn)證碼后輸入")
"""
def get_loop_url(self,response):
login_loop = response.body.decode('gbk')
pa = r'location\.replace\([\'"](.*?)[\'"]\)'
loop_url = re.findall(pa, login_loop)[0]
print(loop_url)
return[Request(loop_url,meta={'cookiejar':response.meta['cookiejar']},callback=self.after_login)]
def after_login(self,response):
return[Request('http://weibo.com/',meta={'cookiejar':response.meta['cookiejar']},callback=self.parse)]
def parse(self, response):
print(response.body.decode('utf-8'))
