實(shí)戰(zhàn)計(jì)劃0430-石頭的練習(xí)作業(yè)
練習(xí)要求
數(shù)據(jù)來源
數(shù)據(jù)處理
運(yùn)行效果：
抓取數(shù)據(jù)
數(shù)據(jù)處理
實(shí)現(xiàn)代碼
__author__ = 'daijielei'
'''
19課時(shí)練習(xí)糟红，為11課時(shí)練習(xí)的加強(qiáng)版,之前從小豬短租處抓取一堆租房詳情纸兔，19課時(shí)需要將其內(nèi)容保存入DB中
'''
from bs4 import BeautifulSoup #解析html文檔用
import requests #抓取html頁面用
import time
import pymongo
client = pymongo.MongoClient('localhost',27017)
xiaozuDB = client['xiaozuDB']
sheet1 = xiaozuDB['sheet1']
'''
saveToDB
將需要保存的內(nèi)容寫入mongoDB
'''
def saveToDB(data):
sheet1.insert_one(data)
client.close()
'''
showxiaozuDB_payMore500
查看mongoDB中保存的數(shù)據(jù)，篩選出金額500以上的顯示出來
'''
def showxiaozuDB_payMore500():
print(sheet1.count())
for item in sheet1.find({'pay':{'$gte':500}}):#篩選條件姿锭，將金額500以上的篩選出來
print(item)
client.close()
'''
cleanxiaozuDB
清除mangoDB中的數(shù)據(jù)
'''
def cleanxiaozuDB():
sheet1.delete_many({})
print(sheet1.count())
client.close()
'''
##################################################################################################################
######## ########
以下內(nèi)容為第11課時(shí)完成，抓取小豬短租的相關(guān)信息
######## ########
##################################################################################################################
'''
#urls用來制定抓取范圍伯铣，header該處無用處
header = {
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
'Host':'bj.xiaozhu.com'
}
#getlist用來抓取對(duì)應(yīng)網(wǎng)頁上的每個(gè)列表的跳轉(zhuǎn)url
#測(cè)試用url = "http://bj.xiaozhu.com/search-duanzufang-p1-0/"
def getlist(url):
webData = requests.get(url,headers=header)
soup = BeautifulSoup(webData.text,'lxml')
houselists = soup.select('ul > li > a')
for houselist in houselists:
listUrl = str(houselist.get('href'))
if(listUrl.find('http://bj.xiaozhu.com/fangzi/')!=-1):#確認(rèn)鏈接內(nèi)容為詳情頁呻此，則進(jìn)行詳情頁信息抓取
#print(listUrl)
getInfoPage(listUrl)
#getInfoPage用來抓取詳情頁里的相關(guān)信息，如標(biāo)題腔寡、地址焚鲜、金額、房屋圖片放前、擁有人圖片等等
#測(cè)試用url = "http://bj.xiaozhu.com/fangzi/525041101.html"
def getInfoPage(url):
webData = requests.get(url,headers=header)
soup = BeautifulSoup(webData.text,'lxml')
time.sleep(2)
titles = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
address = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')
pays = soup.select('#pricePart > div.day_l > span')
houseimages = soup.select('#curBigImage')
ownerimages = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
ownernames = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
ownerSexs = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span')
for title,addres,pay,houseimage,ownerimage,ownername,ownerSex in zip(titles,address,pays,houseimages,ownerimages,ownernames,ownerSexs):
data = {
'title':title.get_text(),
'addres':addres.get_text(),
'pay': int(pay.get_text()) if str((pay.get_text())).isdigit() else 0,
'houseimage':houseimage.get('src'),
'ownerimage':ownerimage.get('src'),
'ownername':ownername.get_text(),
'ownerSex':'female'
}
if('member_boy_ico' in str(ownerSex['class'])):#該處用來判斷是男性還是女性忿磅，男性是通過標(biāo)簽來區(qū)分
data['ownerSex'] = "male"
print(data)
#saveToDB(data)
def startCatch():
urls = ["http://bj.xiaozhu.com/search-duanzufang-p{}-0/".format(str(i)) for i in range(1,5,1)]
for url in urls:
getlist(url)
#入口，該語句只有直接執(zhí)行時(shí)會(huì)被調(diào)用
if __name__ == '__main__':
#startCatch()
showxiaozuDB_payMore500()
#cleanxiaozuDB()
筆記凭语、總結(jié)葱她、思考：
1、這節(jié)練習(xí)主要的重點(diǎn)有似扔，存儲(chǔ)到數(shù)據(jù)庫
1吨些、創(chuàng)建本地mongoDB的數(shù)據(jù)庫連接
client = pymongo.MongoClient('localhost',27017)
2、創(chuàng)建名為'xiaozuDB'的數(shù)據(jù)庫
xiaozuDB = client['xiaozuDB']
3炒辉、創(chuàng)建名為'sheet1'的集合（collection豪墅，和表類似，但是和表對(duì)比靈活多了）
sheet1 = xiaozuDB['sheet1']
4黔寇、往'sheet1'的表中寫入數(shù)據(jù)偶器，這里是直接把字典寫進(jìn)去
sheet1.insert_one(data)
2、篩選出金額為500以上的數(shù)據(jù)
1缝裤、從'sheet1'中篩選出數(shù)據(jù)屏轰，find函數(shù)用來尋找所有滿足條件的數(shù)據(jù)，傳入字典來包含尋找的條件
for item in sheet1.find({'pay':{'$gte':500}}):#篩選條件倘是，將金額500以上的篩選出來
print(item)
