Scrapy登錄知乎要解決兩個(gè)問題
1、session的傳遞式撼，保證處理登錄是同一個(gè)狀態(tài)童社。
2、首個(gè)登錄頁面的改變著隆，由直接爬取的頁面變?yōu)榈卿涰撁嫒怕ィ偃ヅ廊№撁妗?/p>

上代碼

# -*- coding: utf-8 -*-
import scrapy
import re
import json


class ZhihuSpider(scrapy.Spider):
    name = "zhihu"
    allowed_domains = ["www.zhihu.com"]
    start_urls = ['http://www.zhihu.com/']
    headers = {
        "Accept": 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        "Host": 'www.zhihu.com',
        "User-Agent": 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
    }

    def parse(self, response):
        text = response.text
        with open("index_page.html", 'wb') as f:
            f.write(response.text.encode('utf-8'))
            f.close()

    def start_requests(self):
        return [scrapy.Request('https://www.zhihu.com/#signin', headers=self.headers, callback=self.login)]

    def login(self, response):
        response_text = response.text
        match_obj = re.match('.*name="_xsrf" value="(.*)"/>', response_text, re.S)
        xsrf = ''
        if match_obj:
            xsrf = match_obj.group(1)
        if xsrf:
            post_data = {
                "_xsrf": xsrf,
                "email": '',
                "password": ''
            }
            import time
            t = str(int(time.time()*1000))
            captcha_url = "https://www.zhihu.com/captcha.gif?r={}&type=login".format(t)
            yield scrapy.Request(captcha_url, headers=self.headers, meta={'post_data': post_data}, callback=self.login_after_capthcha)

    def login_after_capthcha(self, response):

        with open('captpcha.jpg', 'wb') as f:
            f.write(response.body)
            f.close()
        from PIL import Image
        try:
            im = Image.open('captpcha.jpg')
            im.show()
            im.close()
        except:
            pass
        captcha = input("輸入驗(yàn)證碼")
        post_data = response.meta.get("post_data", {})
        post_url = "https://www.zhihu.com/login/email"
        post_data['captcha'] = captcha
        return [scrapy.FormRequest(
            url=post_url,
            formdata=post_data,
            headers=self.headers,
            callback=self.check_login
        )]

    def check_login(self, response):
        # 驗(yàn)證服務(wù)器返回?cái)?shù)據(jù)判斷是否成功
        text_json = json.loads(response.text)
        if 'msg' in text_json and text_json["msg"] == "登錄成功":
            for url in self.start_urls:  # 從繼承的Spider類中拿的內(nèi)容，恢復(fù)到正確執(zhí)行美浦。
                yield scrapy.Request(url, dont_filter=True, headers=self.headers)

首先對scrapy.Spider類中的start_requests(self)進(jìn)行重載弦赖，改變首先要處理的頁面為登錄頁面。
得到登錄頁面后浦辨，獲得xsrf蹬竖，并下載驗(yàn)證碼，通過scrapy.FormRequest構(gòu)造登錄數(shù)據(jù)流酬，通過check_login回調(diào)函數(shù)判斷登錄是否成功币厕。在代碼的最后一行轉(zhuǎn)回正常的登錄流程。