xpath的使用

#xpath:可以在xml中查找信息，對(duì)xml文檔中元素進(jìn)行遍歷和屬性的提取

# xml:被設(shè)計(jì)的目的是為了傳輸數(shù)據(jù),結(jié)構(gòu)和html非常相識(shí),是一種標(biāo)記語(yǔ)言

"""
xpath常見的語(yǔ)法：
nodename 選取此節(jié)點(diǎn)的所有子節(jié)點(diǎn)
/        從根節(jié)點(diǎn)開始查找
//       匹配節(jié)點(diǎn)媳叨，不考慮節(jié)點(diǎn)的位置
.        選取當(dāng)前節(jié)點(diǎn)
..       選取當(dāng)前節(jié)點(diǎn)的父節(jié)點(diǎn)
a/@href        取標(biāo)簽的數(shù)據(jù)
a/text()       取標(biāo)簽的文本
a[@class="123"] 根據(jù)class屬性尋找標(biāo)簽
a[@id="123"]    根據(jù)id屬性尋找標(biāo)簽

a[@id="123"][last()]  取最后一個(gè)id為123的a標(biāo)簽
a[@id="123"][postion() < 2]  取id為123的前兩個(gè)a標(biāo)簽
"""
import requests
from lxml.html import etree
import re

#樣例()
#http://www.budejie.com/audio/1
#http://www.budejie.com/audio/2
#http://www.budejie.com/audio/3

def load_page_data(url):
    """
    下載器（根據(jù)分頁(yè)url獲取分頁(yè)的頁(yè)面源碼）
    :param url: 分頁(yè)url地址
    :return:
    """
    req_header = {
        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
    }
    response = requests.get(url,headers=req_header)
    if response.status_code == 200:
        print('請(qǐng)求成功')
        # with open('page.html','w') as file:
        #     file.write(response.text)
        status = parse_page_data(response.text)

        if status:
            #請(qǐng)求下一頁(yè)數(shù)據(jù)
            pattern = re.compile('\d+')
            cur_page = re.search(pattern,response.url).group()
            next_page = int(cur_page)+1
            next_page_url = re.sub(pattern,str(next_page),response.url)

            load_page_data(next_page_url)



def parse_page_data(html):
    """
    使用xpath從html頁(yè)面源碼中提取數(shù)據(jù)
    :param html:
    :return:
    """
    #pip3 install lxml
    #使用etree.HTML()方法將html轉(zhuǎn)為xml(element對(duì)象)
    html_element = etree.HTML(html)

    autio_list = html_element.xpath('//div[@class="j-r-c"]/div[@class="j-r-list"]/ul/li')
    print(autio_list)
    print(len(autio_list))

    for autio in autio_list:
        autio_data = {}
        #取出標(biāo)題
        autio_data['name'] = autio.xpath('.//a[@class="u-user-name"]/text()')[0]
        #取出內(nèi)容
        autio_data['content'] = autio.xpath('.//div[@class="j-r-list-c-desc"]/text()')[0]
        #發(fā)布時(shí)間
        autio_data['publishTime'] = autio.xpath('.//span[@class="u-time  f-ib f-fr"]/text()')[0]
        #點(diǎn)贊數(shù)
        autio_data['zanNum'] = autio.xpath('.//li[@class="j-r-list-tool-l-up"]/span/text()')[0]
        #差評(píng)書
        autio_data['lowNum'] = autio.xpath('.//li[@class="j-r-list-tool-l-down "]/span/text()')[0]
        #封面
        autio_data['coverImage'] = autio.xpath('.//div[@class=" j-audio"]/@data-poster')[0]
        #音頻
        autio_data['url'] = autio.xpath('.//div[@class=" j-audio"]/@data-mp3')[0]
        print(autio_data)
        download_audio_by_url(autio_data['url'],autio_data)

    if len(autio_list) > 0:
        return True
    else:
        return False

def download_audio_by_url(url,audioData):
    """
    根據(jù)銀屏url地址下載銀屏數(shù)據(jù)
    :param url:
    :param audioData:
    :return:
    """
    req_header = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
    }
    response = requests.get(url,headers=req_header)
    if response.status_code == 200:
        print(response.url,'下載成功')
        filename = response.url[-17:]
        #print('hhhhhhhhhh',filename)
        print('baisibudejie/'+filename)
        with open('baisibudejie/'+filename,'wb') as file:
            file.write(response.content)
            audioData['localpath'] = 'baisibudejie/'+filename
        #將數(shù)據(jù)存儲(chǔ)到數(shù)據(jù)庫(kù)
        save_data_to_db(audioData)

def save_data_to_db(audio):
    print(audio)


if __name__ == '__main__':

    start_url = 'http://www.budejie.com/audio/1'
    load_page_data(start_url)