案例目的：爬取58同城寵物狗信息
難點(diǎn) - 自動(dòng)代理：爬取58信息前，自動(dòng)設(shè)置 http://www.xicidaili.com/nn 代理網(wǎng)站公布的免費(fèi)代理。如果代理不可用，自動(dòng)更換代理設(shè)置。

成果：

Paste_Image.png

源代碼：

import bs4
import requests
import requests.exceptions
import time
import re

heads = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
}

def get_detail(url):
    soup = get_soup_from_url(url,True)

    title = soup.find("div", class_="mainTitle_content").h1.get_text()
    price = soup.find("span",class_="price c_f50").get_text()
    age = soup.find("span",class_="nianling").get_text().strip()
    breed = soup.find("div",class_="su_tit",text=re.compile(r"品\s*種")).next_sibling.next_sibling.span.get_text().strip()
    contactor = soup.find("span",class_="lianxiren").a.get_text()

    print(price,age,breed,contactor,title)
    time.sleep(1)

def get_list(url):

    def atag_exclude_adds(tag):
        return tag.name == "a" and tag.has_attr("class") and "t" in tag["class"] and "xct" not in tag.parent["class"]

    soup = get_soup_from_url(url,True)
    detail_urls = [i.get("href") for i in soup.find_all(atag_exclude_adds)]

    # print("detail_urls ready")
    for i in detail_urls:
        get_detail(i)
        # print(i)

def get_soup_from_url(url,auto_proxy = False):
    if auto_proxy:
        # proxy = my_proxy.get_current_proxy()
        proxy = my_proxy.get_current_proxy()
    else:
        proxy = {}

    while True:
        # if proxy: print("Trying Proxy:%s" % proxy["http"])
        try:
            respond = requests.get(url,headers=heads,proxies = proxy,timeout=8)
        except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
            print("Proxy %s isn't working well..." % proxy["http"])
            proxy = my_proxy.get_next_proxy()
        else:
            if respond.status_code!= 200:
                print("respond status code problem: %s" % respond.status_code)
                proxy = my_proxy.get_next_proxy()
                continue
            break

    soup = bs4.BeautifulSoup(respond.text,'lxml')
    return soup

class MyProxy(object):
    def __init__(self):
        self.proxy_list = []
        self.refresh_ip_list()
        self.__next__()

    def refresh_ip_list(self):
        print("Refreshing MyProxy IP List...")
        soup = get_soup_from_url("http://www.xicidaili.com/nn/")

        #正則尋找IP地址
        re_ip = re.compile(
            r"(?:(?:25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(?:25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))")
        ip_tag = soup.find_all("td", text=re_ip)  # get_text()

        #獲得IP端口
        ports = soup.select("tr > td:nth-of-type(3)")

        #獲取Proxy速度
        speed_work = [i.get("class")[1] for i in soup.select("tr > td:nth-of-type(7) > div > div")]

        #獲取Proxy連接速度
        speed_connect = [i.get("class")[1] for i in soup.select("tr > td:nth-of-type(8) > div > div")]

        #合并所有數(shù)據(jù)至ip_data : {"ip":XX.XX.XX.XX,"speed_work":"fast","speed_connect":"medium"}
        ip_data = []
        for ip, workspeed, connectspeed,port in zip(ip_tag, speed_work, speed_connect,ports):
            data = {
                "ip": "{ip}:{port}".format(ip = ip.get_text(),port = port.get_text()),
                "speed_work": workspeed,
                "speed_connect": connectspeed
            }
            ip_data.append(data)

        #篩選出速度快的Proxy
        ip_data_filtered = list(filter(lambda i: i["speed_work"] == "fast" and i["speed_connect"] == "fast", ip_data))
        self.proxy_list = [i["ip"] for i in ip_data_filtered]
        self.proxy_list = iter(self.proxy_list)

    def __iter__(self):
        return self

    def __next__(self):
        print("Switching IP Address...")
        try:
            self.currentIP = next(self.proxy_list)
            print("New Proxy: %s" % (self.currentIP))
            return self.currentIP
        except StopIteration as e:
            self.refresh_ip_list()
            return self.__next__()

    def get_current_proxy(self):
        return {"http": self.currentIP}

    def get_next_proxy(self):
        return {"http": self.__next__()}

my_proxy = MyProxy()

dog_lists = ["http://sh.58.com/dog/pn{}".format(i) for i in range(5)]
for list_url in dog_lists:
    print("Scrawling %s" % (list_url))
    get_list(list_url)

總結(jié)：

• soup.find_all()可以使用方法作為參數(shù)诺苹，實(shí)現(xiàn)完全自定義。
• MyProxy 為迭代器雹拄，無(wú)窮迭代收奔，當(dāng)proxy服務(wù)器列表用完時(shí)，刷新網(wǎng)頁(yè)獲得新列表
• soup 中tag的next_sibling通常是空行滓玖。所以表面上相鄰的tag坪哄，需要寫(xiě)成tag.next_sibling.next_sibling來(lái)訪問(wèn)
• requests.get()可以設(shè)置timeout參數(shù)作為等待秒數(shù)。 同時(shí)需要抓取requests.exceptions.ReadTimeout
• ip 地址的正則表達(dá)式非常復(fù)雜势篡，直接搬運(yùn)網(wǎng)上的：r"(?:(?:25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d))).){3}(?:25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))")
• 代理服務(wù)器網(wǎng)址必須帶有端口损姜，否則無(wú)法工作