1、爬蟲一些知識
(1)節(jié)點選擇語法
XPath使用路徑表達(dá)式來選取XML文檔中的節(jié)點或者節(jié)點集罐盔。這些路徑表達(dá)式和我們在常規(guī)的電腦文件系統(tǒng)中看到的表達(dá)式非常相似担平。
nodename：選取此節(jié)點的所有子節(jié)點；
/：從根節(jié)點選让⒙省囤耳；
//：從匹配選擇的當(dāng)前節(jié)點選擇文檔中的節(jié)點，而不考慮它們的位置偶芍；
.：選取當(dāng)前節(jié)點充择；
..：選取當(dāng)前節(jié)點的父節(jié)點；
@：選取屬性匪蟀。
(2)Requests
注：無requests即下載安裝：pip install requests
response = requests.get(url)
response的常用方法：
①response.text：獲取str類型的響應(yīng)
②response.content：獲取bytes類型的響應(yīng)
③response.status_code：獲取狀態(tài)碼
④response.headers：獲取響應(yīng)頭
⑤response.request：獲取響應(yīng)對應(yīng)的請求
(3)為什么請求需要帶上header椎麦？
模擬瀏覽器，欺騙服務(wù)器材彪，獲取和瀏覽器一致的內(nèi)容
headers的形式：字典
2观挎、爬蟲——提取本地html中的數(shù)據(jù)
(1)建立一個index.html文件
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<h1>歡迎來到王者榮耀琴儿！</h1>
<ul>
<li><a ><img src="https://game.gtimg.cn/images/yxzj/img201606/heroimg/508/508.jpg" alt="">伽羅</a></li>
<li>蘇烈</li>
<li>孫策</li>
<li>大喬</li>
</ul>
<ol>
<li>射手</li>
<li>坦克</li>
<li>戰(zhàn)士</li>
<li>輔助</li>
</ol>
<!-- div + css 布局 -->
<div>這是div標(biāo)簽</div>
<div id="container">
<p>被動：伽羅的普攻與技能傷害將會優(yōu)先對目標(biāo)的護(hù)盾效果造成一次等額的傷害</p>
<a >點擊跳轉(zhuǎn)至百度</a>
</div>
<div>這是第二個div標(biāo)簽</div>
</body>
</html>
(2)讀取html文件——使用xpath語法進(jìn)行提取
注：沒有l(wèi)xml即下載安裝：pip install lxml
from lxml import html
# 讀取html文件
with open('./index.html', 'r', encoding='utf-8') as f:
html_date = f.read()
# print(html_date)
# 解析html文件，獲得selector對象
selector = html.fromstring(html_date)
# selector中調(diào)用xpath方法
# 要獲取標(biāo)簽中的內(nèi)容嘁捷，末尾要添加text()
h1 = selector.xpath('/html/body/h1/text()')
print(h1[0])
# // 可以代表從任意位置出發(fā)
# //標(biāo)簽1[@屬性=屬性值]/標(biāo)簽2[@屬性=屬性值]..../text()
a = selector.xpath('//div[@id="container"]/a/text()')
print(a)
# 獲取 p標(biāo)簽的內(nèi)容
p = selector.xpath('//div[@id="container"]/p/text()')
print(p)
# 獲取屬性
link = selector.xpath('//div[@id="container"]/a/@href')
print(link[0])
index.png
3造成、爬蟲——Requests
代碼塊1：
import requests
url = 'https://www.baidu.com'
response = requests.get(url)
print(response)
print(response.text) # 獲取str類型的響應(yīng)
print(response.content) # 獲取bytes類型的響應(yīng)
print(response.headers) # 獲取響應(yīng)頭
print(response.status_code) # 獲取狀態(tài)碼
1.png
代碼塊2：編碼方式
import requests
url = 'https://www.baidu.com'
response = requests.get(url)
print(response.encoding)
baidu.png
import requests
url = 'https://www.taobao.com'
response = requests.get(url)
print(response.encoding)
taobao.png
import requests
url = 'http://www.dangdang.com'
response = requests.get(url)
print(response.encoding)
dangdang.png
代碼塊3：響應(yīng)頭
# 200 ok、404 網(wǎng)頁不存在雄嚣、500 服務(wù)器錯誤晒屎、503 服務(wù)器超時
# 沒有添加請求頭的知乎網(wǎng)站
# resp = requests.get('https://www.zhihu.com')
# print(resp.status_code) # 顯示400
# 使用字典定義請求頭
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get('https://www.zhihu.com', headers = headers)
print(resp.status_code)
# 顯示200
4、爬蟲——dangdang.com
import requests
from lxml import html
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def spider_dangdang(isbn):
book_list = []
# 目標(biāo)站點地址
url = 'http://search.dangdang.com/?key={}&act=input'.format(isbn)
# print(url)
# 獲取站點str類型的響應(yīng)
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get(url, headers=headers)
html_data = resp.text
# 將html頁面寫入本地
with open('dangdang.html', 'w', encoding='utf-8') as f:
f.write(html_data)
# 提取目標(biāo)站的信息
selector = html.fromstring(html_data)
ul_list = selector.xpath('//div[@id="search_nature_rg"]/ul/li')
print('您好缓升，共有{}家店鋪售賣此圖書'.format(len(ul_list)))
# 遍歷 ul_list
for li in ul_list:
# 圖書名稱
title = li.xpath('./a/@title')[0].strip()
# print(title)
# 圖書購買鏈接
link = li.xpath('./a/@href')[0]
# print(link)
# 圖書價格
price = li.xpath('./p[@class="price"]/span[@class="search_now_price"]/text()')[0]
price = float(price.replace('￥', ''))
# print(price)
# 圖書賣家名稱
store = li.xpath('./p[@class="search_shangjia"]/a/text()')
# if len(store) == 0:
# store = '當(dāng)當(dāng)自營'
# else:
# store = store[0]
store = '當(dāng)當(dāng)自營' if len(store) == 0 else store[0]
# print(store)
# 添加每一個商家的圖書信息
book_list.append({
'title':title,
'price':price,
'link':link,
'store':store
})
# 按照價格進(jìn)行排序
book_list.sort(key=lambda x:x['price'])
# 遍歷book_list
for book in book_list:
print(book)
# 展示價格最低的前10家 柱狀圖
# 店鋪的名稱
top10_store = [book_list[i] for i in range(10)]
# x = []
# for store in top10_store:
# x.append(store['store'])
x = [x['store'] for x in top10_store]
print(x)
# 圖書的價格
y = [x['price'] for x in top10_store]
print(y)
# plt.bar(x, y)
# plt.bar(x, y, rotation=75)
plt.barh(x, y) # 橫向
plt.show()
# 存儲成csv文件
df = pd.DataFrame(book_list)
df.to_csv('dangdang.csv')
spider_dangdang('9787115428028') # 調(diào)用
['童心悅美圖書專營店', '當(dāng)當(dāng)自營', '百億德舊書專營店', '鑫源圖書專營店', '童心悅美圖書專營店', '暉文錦繡圖書專營店', '陽光瑞盛圖書專營店', '閱微閣圖書專營店', '金種子圖書專營店', '三味書屋圖書專營店']
[37.0, 62.0, 66.0, 66.1, 75.0, 80.1, 82.4, 86.0, 87.0, 87.0]
image.png
image.png
image.png
dangdang.html.png
dangdang.csv.png
5鼓鲁、爬蟲——練習(xí)
電影名，上映日期仔沿，類型坐桩，上映國家，想看人數(shù)
根據(jù)想看人數(shù)進(jìn)行排序
繪制即將上映電影想看人數(shù)占比圖和即將上映電影國家的占比圖
繪制top5最想看的電影
import requests
from lxml import html
from wordcloud import WordCloud
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def spider_movie(isbn):
movie_list = []
url = 'https://movie.douban.com/cinema/later/{}'.format(isbn)
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get(url, headers=headers)
html_data = resp.text
selector = html.fromstring(html_data)
div_list = selector.xpath('//div[@id="showing-soon"]/div')
print('共有{}部電影即將上映'.format(len(div_list)))
for div in div_list:
# 電影名
name = div.xpath('./div[@class="intro"]/h3/a/text()')[0]
# print(name)
# 上映日期
day = div.xpath('./div[@class="intro"]/ul/li/text()')[0]
# print(day)
# 類型
type = div.xpath('./div[@class="intro"]/ul/li/text()')[1]
# print(type)
# 上映國家
country = div.xpath('./div[@class="intro"]/ul/li/text()')[2]
# print(country)
# 想看人數(shù)
div_three = div.xpath('./div[@class="intro"]/ul/li')[3]
number = div_three.xpath('./span/text()')[0]
number = str(number).replace('人想看', '')
number = int(number)
# print(number)
# 添加電影信息
movie_list.append({
'name':name,
'day':day,
'type':type,
'country':country,
'number':number
})
# 排序
movie_list.sort(key=lambda x:x['number'], reverse=True)
# 遍歷
for movie in movie_list:
print(movie)
# 繪制即將上映電影最想看前五人數(shù)占比圖
top5_movie = [movie_list[i] for i in range(5)]
labels = [x['name'] for x in top5_movie]
# print(labels)
counts = [x['number'] for x in top5_movie]
# print(counts)
colors = ['red', 'purple', 'yellow', 'gray', 'green']
plt.pie(counts, labels=labels, autopct='%1.2f%%', colors=colors)
plt.legend(loc=2)
plt.axis('equal')
plt.show()
# 繪制即將上映電影國家的占比圖
total = [x['country'] for x in movie_list]
text = ''.join(total)
print(text)
words_list = jieba.lcut(text)
print(words_list)
counts = {}
excludes ={"大陸"}
for word in words_list:
if len(word) <= 1:
continue
else:
counts[word] = counts.get(word, 0) + 1
print(counts)
for word in excludes:
del counts[word]
items = list(counts.items())
items.sort(key=lambda x: x[1], reverse=True)
numm = [] # 數(shù)量
labels = [] # 國家
for i in range(len(items)):
x, y = items[i]
numm.append(y)
if(x == "中國"):
x = "中國大陸"
labels.append(x)
plt.pie(numm, labels=labels, autopct='%1.2f%%')
plt.legend(loc=2)
plt.axis('equal')
plt.show()
# top5.png
text = ' '.join(labels)
WordCloud(
font_path='MSYH.TTC',
background_color='white',
width=800,
height=600,
collocations=False
).generate(text).to_file('top5.png')
spider_movie('chongqing')
image.png
image.png
image.png
top5.png
