爬蟲
- 在Gogle瀏覽器上安裝Xpath Helper插件
- 實(shí)例：爬圖書的價(jià)格吴汪，排序等
import requests
from lxml import html
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def spider_dangdang(isbn):
book_list = []
# 目標(biāo)站點(diǎn)地址
url = 'http://search.dangdang.com/?key={}&act=input'.format(isbn)
# print(url)
# 獲取站點(diǎn)str類型的響應(yīng)
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get(url, headers=headers)
html_data = resp.text
# 將html頁面寫入本地
# with open('dangdang.html', 'w', encoding='utf-8') as f:
# f.write(html_data)
# 提取目標(biāo)站的信息
selector = html.fromstring(html_data)
ul_list = selector.xpath('//div[@id="search_nature_rg"]/ul/li')
print('您好洲押，共有{}家店鋪售賣此圖書'.format(len(ul_list)))
# print('qwertyui', ul_list)
# 遍歷 ul_list
for li in ul_list:
# 圖書名稱
title = li.xpath('./a/@title')[0].strip()
print('title---', title)
# 圖書購(gòu)買鏈接
link = li.xpath('a/@href')[0]
# print(link)
# 圖書價(jià)格
price = li.xpath('./p[@class="price"]/span[@class="search_now_price"]/text()')[0]
#print(price)
price = float(price.replace('￥',''))
print(price)
# 圖書賣家名稱
store = li.xpath('./p[@class="search_shangjia"]/a/text()')
store = '當(dāng)當(dāng)自營(yíng)' if len(store) == 0 else store[0]
# print(store)
# 添加每一個(gè)商家的圖書信息
book_list.append({
'title':title,
'price':price,
'link':link,
'store':store
})
# 按照價(jià)格進(jìn)行排序
book_list.sort(key=lambda x:x['price'])
# 遍歷booklist
for book in book_list:
print(book)
# 展示價(jià)格最低的前10家 柱狀圖
# 店鋪的名稱
top10_store = [book_list[i] for i in range(10)]
# x = []
# for store in top10_store:
# x.append(store['store'])
x = [x['store'] for x in top10_store]
print(x)
# 圖書的價(jià)格
y = [x['price'] for x in top10_store]
print(y)
# plt.bar(x, y)
plt.barh(x, y)
plt.show()
# 存儲(chǔ)成csv文件
df = pd.DataFrame(book_list)
df.to_csv('dangdang.csv')
spider_dangdang('9787115428028')
- 作業(yè)：爬電影網(wǎng)站设塔，得到電影名煮盼、想看人數(shù)等信息短纵，繪制分析圖
import requests
from lxml import html
import pandas as pd
from matplotlib import pyplot as plt
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def spider_douban():
movie_list = []
# 目標(biāo)站點(diǎn)地址
url = 'https://movie.douban.com/cinema/later/chongqing/?qq-pf-to=pcqq.group'
print(url)
# 添加頭文件，偽裝成瀏覽器僵控，防止被發(fā)現(xiàn)
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
resp = requests.get(url, headers=headers)
html_data = resp.text
selector = html.fromstring(html_data)
ul_list = selector.xpath("http://div[@id='showing-soon']/div")
print('重慶即將上映的電影有',len(ul_list),'部') #打印有多少部電影
# 遍歷 ul_list
for li in ul_list:
i = 0
# i = 0
# 電影名
name_movie = li.xpath("./div/h3/a/text()")[0]
# print('電影名', name_movie)
# 上映日期
line_data = li.xpath("./div/ul/li[1]/text()")[i]
# print('上映日期', line_data)
# 類型
movie_type = li.xpath("./div/ul/li[2]/text()")[i]
# print('電影類型', movie_type)
# 上映國(guó)家
movie_city = li.xpath("./div/ul/li[3]/text()")[i]
# print('上映國(guó)家', movie_city)
# 想看人數(shù)
movie_wantnum = li.xpath("./div/ul/li/span/text()")[i]
# print('想看人數(shù)', movie_wantnum)
movie_wantnum = int(movie_wantnum.replace('人想看', ''))
movie_list.append({
'電影名': name_movie,
'上映日期': line_data,
'類型': movie_type,
'上映國(guó)家': movie_city,
'想看人數(shù)': movie_wantnum,
})
i += 1
movie_list.sort(key=lambda a: a['想看人數(shù)'], reverse=True)
print(movie_list)
movie_city = []
for i in movie_list:
movie_city.append(i['上映國(guó)家'])
print('上映國(guó)家', movie_city)
################# 繪制上映國(guó)家云詞
text = ' '.join(movie_city)
print(text)
from wordcloud import WordCloud
import imageio
mask = imageio.imread('./image/china.jpg')
WordCloud(
font_path='msyh.ttc',
background_color='black',
width=800,
height=600,
collocations=False, # 相鄰兩個(gè)重復(fù)詞之間的匹配
mask=mask
).generate(text).to_file('上映國(guó)家.png')
# 繪制 上映國(guó)家占比
from random import randint
from matplotlib import pyplot as plt
# 解決亂碼
plt.rcParams["font.sans-serif"] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
import numpy as np
counts = {} # 上映國(guó)家和人數(shù)
lli = [] # 上映國(guó)家
lii = [] # 每個(gè)國(guó)家出現(xiàn)的次數(shù)
for i in movie_city:
counts[i] = counts.get(i, 0)+1
print(counts)
items = list(counts.items()) # 將字典轉(zhuǎn)換為列表
print(len(items))
items.sort(key=lambda x: x[1], reverse=True) # 使用函數(shù)從大到小排序
for i in range(len(items)): # 有多少個(gè)國(guó)家循環(huán)多少次香到，解包 出國(guó)家和國(guó)家的次數(shù)
role, count = items[i] # 序列解包
lli.append(count)
lii.append(role)
# for _ in range(count):
# li.append(role)
# print(li)
print(lii)
print(lli)
explode = [0.1, 0, 0, 0]
plt.pie(lli, explode=explode, shadow=True, labels=lii, autopct='%1.1f%%')
# top5_store = [movie_list[i] for i in range(5)]
name_movie = [] #電影名
for i in movie_list:
name_movie.append(i['電影名'])
if len(name_movie) >= 5:
break
print('電影名', name_movie)
# 想看人數(shù)
movie_wantnum = []
for i in movie_list:
movie_wantnum.append(i['想看人數(shù)'])
if len(movie_wantnum) >= 5:
break
print('想看人數(shù)', movie_wantnum)
plt.barh(name_movie, movie_wantnum)
plt.show()
spider_douban()
# # # 獲取str類型的響應(yīng)
# # print(response.text)
# # # 獲取bytes類型的響應(yīng)
# # print(response.content)
# # # 獲取響應(yīng)頭
# # print(response.headers)
# # # 獲取狀態(tài)碼
# # print(response.status_code)
上映國(guó)家云圖.PNG
上映國(guó)家占比.PNG
想看人數(shù)TOP5電影.PNG
