import random
from collections import defaultdict

from scrapy import signals
from twisted.internet.error import ConnectionRefusedError, TimeoutError

class QczjproDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    # 初始化數(shù)據(jù)
    def __init__(self, settings):
        self.settings = settings
        self.ips = settings.getlist("IPS")
        #錯(cuò)誤記錄   轉(zhuǎn)成字典結(jié)構(gòu)（value是int類型） 默認(rèn)為0
        self.proxy_error_stats = defaultdict(int)

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        # 實(shí)例化的時(shí)候垄琐，把settings給init函數(shù)
        s = cls(crawler.settings)
        # 實(shí)例化的時(shí)候注冊(cè)了個(gè)信號(hào)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        # 加代理IP
        if "proxy" not in request.meta:
            proxy = random.choice(self.ips)
            # proxy = random.choice(self.cookies)
            # proxy = random.choice(self.headers)
            print(proxy, "---------------------------------")
            request.meta['proxy'] = proxy
        # else:
        #     print(request.meta["proxy"])
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        status_code = response.status
        if status_code in [403, 500, 404]:

            ip = request.meta["proxy"]
            # 統(tǒng)計(jì)某個(gè)IP的錯(cuò)誤次數(shù)
            self.proxy_error_stats[ip] += 1
            # 如果大于我們約定的五次，那就刪IP
            if self.proxy_error_stats[ip] > 5:
                # self.ips.remove(ip)
                self.remove_fail_ip(ip)
                # 去掉這個(gè)請(qǐng)求的meta的proxy
                del request.meta["proxy"]
            return request
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        # 當(dāng)出現(xiàn)跟代理IP有關(guān)系的異常的時(shí)候
        if isinstance(exception, (ConnectionRefusedError, TimeoutError)):

            # self.ips 去掉這個(gè)無(wú)效的IP
            ip = request.meta["proxy"]
            # self.ips.remove(ip)
            self.remove_fail_ip(ip)
            # 去掉這個(gè)請(qǐng)求的meta的proxy
            del request.meta["proxy"]
            # 重新安排下載
            return request

    def remove_fail_ip(self, ip):
        if ip in self.ips:
            self.ips.remove(ip)

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

注：另外要在settings里面加一個(gè)IPS列表绘迁，里面放代理ip