距離上次爬蟲有一段時(shí)間了恬涧，這次就用requests做點(diǎn)事兒糖荒，爬取一個(gè)網(wǎng)頁(yè)并且把數(shù)據(jù)添加到數(shù)據(jù)庫(kù)中带斑，酷不酷阱州？
首先聲明使用技術(shù)棧：
Python3 : python3+ 版本
requests: python的一個(gè)http請(qǐng)求庫(kù)
xpath : 用于將html轉(zhuǎn)換為xml,解析xml
pymongo: 連接mongodb數(shù)據(jù)庫(kù),儲(chǔ)存數(shù)據(jù)
-
requests+xpath+pymongo
直接上代碼
import requests
from lxml import etree
from pymongo import MongoClient
# 建立mongo
client = MongoClient('mongodb://localhost:27017/')
db = client['mydb']
coupon = db['coupon']
startUrl = "https://www.haoshsh.com/jiu/index/cid/4/p/1.html"
headerUrl = "https://www.haoshsh.com"
data = []
def VisitUrl(url):
r = requests.get(url)
r.encoding="UTF-8"
html = r.text
xmlContene = etree.HTML(html)
nextUrl = xmlContene.xpath('//div[@class="page"]/div/a[text()="下一頁(yè)"]/@href')
if len(nextUrl)!=0:
nextUrl = xmlContene.xpath('//div[@class="page"]/div/a[text()="下一頁(yè)"]/@href')[0]
list = xmlContene.xpath('//ul[@class="goods-list clear"]/li')
for index in range(len(list)):
value = list[index]
# # title
title = value.xpath('./div/h3/a/text()')[0]
# # 原價(jià)
OriginalPrice = value.xpath('./div/div[@class="good-price"]/span[@class="des-other"]/span[@class="price-old"]/text()')[0]
# # 券后價(jià)
PostPrice = value.xpath('./div/div[@class="good-price"]/span[@class="price-current"]/text()')[0]
# # 圖片路徑
imgsrc = value.xpath('./div/div[@class="good-pic"]/a/img/@d-src')[0]
json = {
"title":title,
"OriginalPrice":OriginalPrice,
"PostPrice":PostPrice,
"imgsrc":imgsrc
}
# 存儲(chǔ)數(shù)據(jù)到數(shù)據(jù)庫(kù)
saveDataBase(json)
if len(nextUrl)==0:
print('\033[1;31m 無(wú)數(shù)據(jù),爬取完畢 \033[0m')
return url
else:
print('\033[1;32m 下一頁(yè)開始爬取 : '+headerUrl+nextUrl+'\033[0m ')
return VisitUrl(headerUrl+nextUrl)
def saveDataBase(data):
coupon.insert_one(data)
VisitUrl(startUrl)
