• XPath是在HTML/XML中查找信息的語言
• lxml模塊

XPath

xpath節(jié)點(diǎn)關(guān)系

每個xml標(biāo)簽都叫做節(jié)點(diǎn)，最頂層的節(jié)點(diǎn)為根節(jié)點(diǎn)
節(jié)點(diǎn)選擇工具
Chrome插件 XPath Helper 密碼:337b

xpath語法

路徑表達(dá)式

查找特定節(jié)點(diǎn)

通配符

選取若干路徑
使用“|”運(yùn)算符連接路徑表達(dá)式

lxml模塊

安裝
pip install lxml
使用

• 導(dǎo)入lxml模塊的etree庫
  from lxml import etree
• 利用etree.HTML造虏，實(shí)例化Element對象
  使用xpath(路徑表達(dá)式)御吞，返回列表

html = etree.HTML(text) 
ret_list = html.xpath("xpath字符串")

作業(yè)：

用XPath來做一個簡單的爬蟲麦箍，爬取某個貼吧里的所有帖子，獲取每個帖子的標(biāo)題陶珠，連接和帖子中圖片

#coding:utf-8
import os
import requests
from lxml import etree

"""
用XPath來做一個簡單的爬蟲挟裂，爬取某個貼吧里的所有帖子，獲取每個帖子的標(biāo)題揍诽，連接和帖子中圖片
"""

class TieBa():

    def __init__(self):
        self.url = "http://tieba.baidu.com/f?ie=utf-8&kw=python&fr=search"
        self.headers = {
            "User-Agent": "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) "
        }

    def get_data(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content

    def parse_data(self, data):
        # 解析:獲取每個帖子的標(biāo)題诀蓉，連接和帖子中圖片
        html = etree.HTML(data)
        dt_list = html.xpath('//ul[@id="thread_list"]/li/div/div/div/div/a')
        data_list = []
        for dt in dt_list:
            temp = {}
            # xpath()返回的是個列表
            temp['title'] = dt.xpath('./text()')[0]
            temp['link'] = 'https://tieba.baidu.com' + dt.xpath('./@href')[0]
            data_list.append(temp)
        # 匹配下一頁的鏈接
        try:
            next_url = 'http:' + html.xpath('//a[text()="下一頁>"]/@href')[0]
        except:
            next_url = None
        return data_list, next_url

    def image_data(self, data):
        html = etree.HTML(data)
        image_list = html.xpath('//*[contains(@id,"post_content_")]/img/@src')
        return image_list

    def download(self, image_list):
        if not os.path.exists('images'):
            os.makedirs('images')
        for image in image_list:
            # os.sep根據(jù)所屬平臺生成分隔符
            filename='images'+os.sep+image.split("/")[-1]
            if "sign" in image:
                data=self.get_data(image)
                with open(filename, 'wb') as f:
                    f.write(data)

    def run(self):
        # url
        # header
        next_url = self.url
        # 循環(huán)
        while True:
            # 發(fā)送請求獲取響應(yīng)
            data = self.get_data(next_url)
            # 解析:獲取每個帖子的標(biāo)題，連接
            data_list, next_url = self.parse_data(data)
            print(data_list)
            # 打開每個帖子詳情頁暑脆，解析渠啤，獲取圖片鏈接，下載
            for data in data_list:
                tz_url=data['link']
                tz_data = self.get_data(tz_url)
                image_list = self.image_data(tz_data)
                print(image_list)
                # 保存
                self.download(image_list)
            # 根據(jù)是否有下一頁判斷是否退出循環(huán)
                if not next_url:
                    break

if __name__ == "__main__":
    baidu = TieBa()
    baidu.run()