實(shí)戰(zhàn)計劃0430-石頭的練習(xí)作業(yè)

練習(xí)的要求

課程要求抓取的內(nèi)容：

實(shí)現(xiàn)效果如下

運(yùn)行結(jié)果

相關(guān)代碼

__author__ = 'daijielei'

'''
11課時練習(xí)奋早，從小豬短租處抓取一堆租房詳情
'''
from bs4 import BeautifulSoup   #解析html文檔用
import requests                 #抓取html頁面用
import time

#urls用來制定抓取范圍夷蚊，header該處無用處
header = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
    'Host':'bj.xiaozhu.com'
}


#getlist用來抓取對應(yīng)網(wǎng)頁上的每個列表的跳轉(zhuǎn)url
#測試用url = "http://bj.xiaozhu.com/search-duanzufang-p1-0/"
def getlist(url):
    webData = requests.get(url,headers=header)
    soup = BeautifulSoup(webData.text,'lxml')

    houselists = soup.select('ul > li > a')

    for houselist in houselists:
        listUrl = str(houselist.get('href'))
        if(listUrl.find('http://bj.xiaozhu.com/fangzi/')!=-1):#確認(rèn)鏈接內(nèi)容為詳情頁娱颊，則進(jìn)行詳情頁信息抓取
            #print(listUrl)
            getInfoPage(listUrl)


#getInfoPage用來抓取詳情頁里的相關(guān)信息傲诵，如標(biāo)題、地址箱硕、金額拴竹、房屋圖片、擁有人圖片等等
#測試用url = "http://bj.xiaozhu.com/fangzi/525041101.html"
def getInfoPage(url):
    webData = requests.get(url,headers=header)
    soup = BeautifulSoup(webData.text,'lxml')
    time.sleep(2)

    titles = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
    address = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')
    pays = soup.select('#pricePart > div.day_l > span')
    houseimages = soup.select('#curBigImage')
    ownerimages = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
    ownernames = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
    ownerSexs = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span')

    for title,addres,pay,houseimage,ownerimage,ownername,ownerSex in zip(titles,address,pays,houseimages,ownerimages,ownernames,ownerSexs):
        data = {
            'title':title.get_text(),
            'addres':addres.get_text(),
            'pay':pay.get_text(),
            'houseimage':houseimage.get('src'),
            'ownerimage':ownerimage.get('src'),
            'ownername':ownername.get_text(),
            'ownerSex':'female'
        }
        if('member_boy_ico' in str(ownerSex['class'])):#該處用來判斷是男性還是女性剧罩，男性是通過標(biāo)簽來區(qū)分
            data['ownerSex'] = "male"
        print(data)

def startCatch():
    urls = ["http://bj.xiaozhu.com/search-duanzufang-p{}-0/".format(str(i)) for i in range(1,3,1)]
    for url in urls:
        getlist(url)

#入口栓拜，該語句只有直接執(zhí)行時會被調(diào)用
if __name__ == '__main__':
    startCatch()

筆記、想法、總結(jié)

1菱属、這個代碼是之前寫的了钳榨，所以顯得比較亂，尤其是抓取的部分現(xiàn)在寫的話我就不會用這種寫法了

2纽门、這里面主要有3個函數(shù)薛耻，功能還是比較清晰，先爬取列表赏陵，再進(jìn)列表里去爬取數(shù)據(jù)：

def startCatch():    #用來便利列表饼齿，會調(diào)用getlist
def getlist(url):      #用來抓取列表的數(shù)據(jù)，會調(diào)用getInfoPage
def getInfoPage(url):#抓取詳情頁的數(shù)據(jù)

3蝙搔、比較難抓的數(shù)據(jù)只有性別信息缕溉，因?yàn)椴皇侵苯幽茏ト〉闹担且鲆幌屡袛啵?br> 在html中為如下,是用span里的class來區(qū)別男女吃型，男為member_boy_ico证鸥、女為member_girl_ico：

<h6>
<a class="lorder_name"  title="ocheese" target="_blank">ocheese</a>
<span class="member_girl_ico"></span>
</h6>

代碼中就是做一下判斷即可

        if('member_boy_ico' in str(ownerSex['class'])):#該處用來判斷是男性還是女性，男性是通過標(biāo)簽來區(qū)分
            data['ownerSex'] = "male"