嗨嘍壳猜,大家好呀~這里是愛看美女的茜茜吶
知識點:
動態(tài)數(shù)據(jù)抓包
requests發(fā)送請求
結(jié)構(gòu)化+非結(jié)構(gòu)化數(shù)據(jù)解析
開發(fā)環(huán)境:
python 3.8 運行代碼
pycharm 2022.3.2 輔助敲代碼 專業(yè)版
requests 發(fā)送請求 pip install requests
parsel 解析數(shù)據(jù) pip install parsel
爬蟲案例實現(xiàn)流程:
一. 抓包分析 (找到數(shù)據(jù)來源)
https://tianqi.2345.com/Pc/GetHistoryareaInfo%5BareaId%5D=57687&areaInfo%5BareaType%5D=2&date%5Byear%5D=2023&date%5Bmonth%5D=3
二. 代碼實現(xiàn)
發(fā)送請求 (訪問 數(shù)據(jù)來源的url)
獲取數(shù)據(jù)
提取數(shù)據(jù) (將需要的信息提取出來)
保存數(shù)據(jù) (保存為csv表格數(shù)據(jù))
獲取數(shù)據(jù)
import requests
import parsel
import csv
import execjs
f = open('demo.js', mode='r', encoding='utf-8').read()
ctx = execjs.compile(f)
city_list = ctx.call("get_city")
areaInfoList = []
for city_li in city_list:
if city_li:
for city_info in city_li:
info_list = city_info.split('|')
for info in info_list:
city = info.split(' ')[-1]
city_name = city.split('-')[0]
city_code = city.split('-')[1]
areaInfoList.append([city_name, city_code])
# print(areaInfoList)
with open('tianqi.csv', mode='a', newline='', encoding='utf-8') as f:
csv.writer(f).writerow(["日期", "最高溫", "最低溫", "天氣", "風(fēng)力風(fēng)向", "空氣質(zhì)量指數(shù)", "城市"])
for areaInfo in areaInfoList:
city_name = areaInfo[0]
city_code = areaInfo[1]
for year in range(2020, 2023):
for month in range(1, 13):
url = f'https://tianqi.2345.com/Pc/GetHistory?areaInfo%5BareaId%5D={city_code}&areaInfo%5BareaType%5D=2&date%5Byear%5D={year}&date%5Bmonth%5D={month}'
# 1. 發(fā)送請求 (訪問 數(shù)據(jù)來源的url)
response = requests.get(url)
# 2. 獲取數(shù)據(jù)
json_data = response.json() # .json()可以將拿到的text內(nèi)容 轉(zhuǎn)為字典數(shù)據(jù)
# 3. 提取數(shù)據(jù) (將需要的信息提取出來)
html_data = json_data.get('data')
# tr
select = parsel.Selector(html_data)
trs = select.xpath('//tr')
for tr in trs[1:]:
tds = tr.xpath('./td//text()').getall()
tds.append(city_name)
print(tds)
with open('tianqi.csv', mode='a', newline='', encoding='utf-8') as f:
csv.writer(f).writerow(tds)
數(shù)據(jù)可視化
導(dǎo)入包
import pandas as pd
import datetime
from pyecharts import options as opts
from pyecharts.charts import *
from pyecharts.commons.utils import JsCode
讀入數(shù)據(jù)
data = pd.read_csv('天氣.csv')
data
數(shù)據(jù)預(yù)覽
data.sample(5)
data.info()
分割日期/星期
data[['日期','星期']] = data['日期'].str.split(' ',expand=True,n=1)
data
去除多余字符
data[['最高溫度','最低溫度']] = data[['最高溫度','最低溫度']].apply(lambda x: x.str.replace('°','').replace('', '0'))
data.head()
計算下雪天氣
data.loc[data['天氣'].str.contains('雪'),'下雪嗎']='是'
data.fillna('否',inplace=True)
分割日期時間
data['日期'] = pd.to_datetime(data['日期'])
data[['最高溫度','最低溫度']] = data[['最高溫度','最低溫度']].astype('int')
data['年份'] = data['日期'].dt.year
data['月份'] = data['日期'].dt.month
data['日'] = data['日期'].dt.day
# 預(yù)覽
data.sample(5)
各城市初雪的時間
s_data = data[data['下雪嗎']=='是']
s_data[(s_data['月份']>=9)].groupby('年份').first().reset_index()
各城市下雪天氣分布
s_data.groupby(['城市','年份'])['日期'].count().to_frame('下雪天數(shù)').reset_index()
做透視表
data_bj = data[(data['年份'] == 2021) & (data['城市'] == '北京')]
data_bj = data_bj.groupby(['月份','天氣'], as_index=False)['日期'].count()
data_pivot = pd.pivot(data_bj,
values='日期',
index='月份',
columns='天氣')
data_pivot = data_pivot.astype('float')
# 按照 索引年月倒序排序
data_pivot.sort_index(ascending=False,inplace=True)
data_pivot
北上廣深2021年10月份天氣熱力圖分布
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import seaborn as sns
#設(shè)置全局默認(rèn)字體 為 雅黑
plt.rcParams['font.family'] = ['Microsoft YaHei']
# 設(shè)置全局軸標(biāo)簽字典大小
plt.rcParams["axes.labelsize"] = 14
# 設(shè)置背景
sns.set_style("darkgrid",{"font.family":['Microsoft YaHei', 'SimHei']})
# 設(shè)置畫布長寬 和 dpi
plt.figure(figsize=(18,8),dpi=100)
# 自定義色卡
cmap = mcolors.LinearSegmentedColormap.from_list("n",['#95B359','#D3CF63','#E0991D','#D96161','#A257D0','#7B1216'])
# 繪制熱力圖
ax = sns.heatmap(data_pivot, cmap=cmap, vmax=30,
annot=True, # 熱力圖上顯示數(shù)值
linewidths=0.5,
)
# 將x軸刻度放在最上面
ax.xaxis.set_ticks_position('top')
plt.title('北京最近10個月天氣分布',fontsize=16) #圖片標(biāo)題文本和字體大小
plt.show()
data_gz= data[(data['年份'] == 2021) & (data['城市'] == '廣州')]
data_gz = data_gz.groupby(['月份','天氣'], as_index=False)['日期'].count()
data_sz= data[(data['年份'] == 2021) & (data['城市'] == '深圳')]
data_sz = data_sz.groupby(['月份','天氣'], as_index=False)['日期'].count()
data_sh= data[(data['年份'] == 2021) & (data['城市'] == '上海')]
data_sh = data_sh.groupby(['月份','天氣'], as_index=False)['日期'].count()
data_pivot_sz = pd.pivot(data_sz,
values='日期',
index='月份',
columns='天氣')
data_pivot_sz = data_pivot_sz.astype('float')
# 按照 索引年月倒序排序
data_pivot_sz.sort_index(ascending=False,inplace=True)
#設(shè)置全局默認(rèn)字體 為 雅黑
plt.rcParams['font.family'] = ['Microsoft YaHei']
# 設(shè)置全局軸標(biāo)簽字典大小
plt.rcParams["axes.labelsize"] = 14
# 設(shè)置背景
sns.set_style("darkgrid",{"font.family":['Microsoft YaHei', 'SimHei']})
# 設(shè)置畫布長寬 和 dpi
plt.figure(figsize=(18,8),dpi=100)
# 自定義色卡
cmap = mcolors.LinearSegmentedColormap.from_list("n",['#95B359','#D3CF63','#E0991D','#D96161','#A257D0','#7B1216'])
# 繪制熱力圖
ax_sz = sns.heatmap(data_pivot_sz, cmap=cmap, vmax=31,
annot=True, # 熱力圖上顯示數(shù)值
linewidths=0.5,
)
# 將x軸刻度放在最上面
ax_sz.xaxis.set_ticks_position('top')
plt.title('深圳最近10個月天氣分布',fontsize=16) #圖片標(biāo)題文本和字體大小
plt.show()
data_pivot_gz = pd.pivot(data_gz,
values='日期',
index='月份',
columns='天氣')
data_pivot_gz = data_pivot_gz.astype('float')
# 按照 索引年月倒序排序
data_pivot_gz.sort_index(ascending=False,inplace=True)
#設(shè)置全局默認(rèn)字體 為 雅黑
plt.rcParams['font.family'] = ['Microsoft YaHei']
# 設(shè)置全局軸標(biāo)簽字典大小
plt.rcParams["axes.labelsize"] = 14
# 設(shè)置背景
sns.set_style("darkgrid",{"font.family":['Microsoft YaHei', 'SimHei']})
# 設(shè)置畫布長寬 和 dpi
plt.figure(figsize=(18,8),dpi=100)
# 自定義色卡
cmap = mcolors.LinearSegmentedColormap.from_list("n",['#95B359','#D3CF63','#E0991D','#D96161','#A257D0','#7B1216'])
# 繪制熱力圖
ax_sz = sns.heatmap(data_pivot_gz, cmap=cmap, vmax=31,
annot=True, # 熱力圖上顯示數(shù)值
linewidths=0.5,
)
# 將x軸刻度放在最上面
ax_sz.xaxis.set_ticks_position('top')
plt.title('廣州最近10個月天氣分布',fontsize=16) #圖片標(biāo)題文本和字體大小
plt.show()
data_pivot_sh = pd.pivot(data_sh,
values='日期',
index='月份',
columns='天氣')
data_pivot_sh = data_pivot_sh.astype('float')
# 按照 索引年月倒序排序
data_pivot_sh.sort_index(ascending=False,inplace=True)
#設(shè)置全局默認(rèn)字體 為 雅黑
plt.rcParams['font.family'] = ['Microsoft YaHei']
# 設(shè)置全局軸標(biāo)簽字典大小
plt.rcParams["axes.labelsize"] = 14
# 設(shè)置背景
sns.set_style("darkgrid",{"font.family":['Microsoft YaHei', 'SimHei']})
# 設(shè)置畫布長寬 和 dpi
plt.figure(figsize=(18,8),dpi=100)
# 自定義色卡
cmap = mcolors.LinearSegmentedColormap.from_list("n",['#95B359','#D3CF63','#E0991D','#D96161','#A257D0','#7B1216'])
# 繪制熱力圖
ax_sz = sns.heatmap(data_pivot_sh, cmap=cmap, vmax=31,
annot=True, # 熱力圖上顯示數(shù)值
linewidths=0.5,
)
# 將x軸刻度放在最上面
ax_sz.xaxis.set_ticks_position('top')
plt.title('上海最近10個月天氣分布',fontsize=16) #圖片標(biāo)題文本和字體大小
plt.show()
data_bj = data[(data['城市']=='北京') & (data['年份'] == 2021)]
data_bj['日期'] = pd.to_datetime(data_bj.日期,format="%Y年%m月%d日")
data_bj = data_bj.sort_values(by='日期',ascending=True)
北京2021年每日最高最低溫度變化
'''
python資料獲取看這里噢A晒省默责! 小編 V:Pytho8987(記得好友驗證備注:6 否則可能不通過)
即可獲饶反颉:文章源碼/教程/資料/解答等福利亭病,還有不錯的視頻學(xué)習(xí)教程和PDF電子書逻恐!
'''
color0 = ['#FF76A2','#24ACE6']
color_js0 = """new echarts.graphic.LinearGradient(0, 1, 0, 0,
[{offset: 0, color: '#FFC0CB'}, {offset: 1, color: '#ed1941'}], false)"""
color_js1 = """new echarts.graphic.LinearGradient(0, 1, 0, 0,
[{offset: 0, color: '#FFFFFF'}, {offset: 1, color: '#009ad6'}], false)"""
tl = Timeline()
for i in range(0,len(data_bj)):
coordy_high = list(data_bj['最高溫度'])[i]
coordx = list(data_bj['日期'])[i]
coordy_low = list(data_bj['最低溫度'])[i]
x_max = list(data_bj['日期'])[i]+datetime.timedelta(days=10)
y_max = int(max(list(data_bj['最高溫度'])[0:i+1]))+3
y_min = int(min(list(data_bj['最低溫度'])[0:i+1]))-3
title_date = list(data_bj['日期'])[i].strftime('%Y-%m-%d')
c = (
Line(
init_opts=opts.InitOpts(
theme='dark',
#設(shè)置動畫
animation_opts=opts.AnimationOpts(animation_delay_update=800),#(animation_delay=1000, animation_easing="elasticOut"),
#設(shè)置寬度因悲、高度
width='1500px',
height='900px', )
)
.add_xaxis(list(data_bj['日期'])[0:i])
.add_yaxis(
series_name="",
y_axis=list(data_bj['最高溫度'])[0:i], is_smooth=True,is_symbol_show=False,
linestyle_opts={
'normal': {
'width': 3,
'shadowColor': 'rgba(0, 0, 0, 0.5)',
'shadowBlur': 5,
'shadowOffsetY': 10,
'shadowOffsetX': 10,
'curve': 0.5,
'color': JsCode(color_js0)
}
},
itemstyle_opts={
"normal": {
"color": JsCode(
"""new echarts.graphic.LinearGradient(0, 0, 0, 1, [{
offset: 0,
color: '#ed1941'
}, {
offset: 1,
color: '#009ad6'
}], false)"""
),
"barBorderRadius": [45, 45, 45, 45],
"shadowColor": "rgb(0, 160, 221)",
}
},
)
.add_yaxis(
series_name="",
y_axis=list(data_bj['最低溫度'])[0:i], is_smooth=True,is_symbol_show=False,
# linestyle_opts=opts.LineStyleOpts(color=color0[1],width=3),
itemstyle_opts=opts.ItemStyleOpts(color=JsCode(color_js1)),
linestyle_opts={
'normal': {
'width': 3,
'shadowColor': 'rgba(0, 0, 0, 0.5)',
'shadowBlur': 5,
'shadowOffsetY': 10,
'shadowOffsetX': 10,
'curve': 0.5,
'color': JsCode(color_js1)
}
},
)
.set_global_opts(
title_opts=opts.TitleOpts("北京2021年每日最高最低溫度變化\n\n{}".format(title_date),pos_left=330,padding=[30,20]),
xaxis_opts=opts.AxisOpts(type_="time",max_=x_max),#, interval=10,min_=i-5,split_number=20,axistick_opts=opts.AxisTickOpts(length=2500),axisline_opts=opts.AxisLineOpts(linestyle_opts=opts.LineStyleOpts(color="grey"))
yaxis_opts=opts.AxisOpts(min_=y_min,max_=y_max),#坐標(biāo)軸顏色,axisline_opts=opts.AxisLineOpts(linestyle_opts=opts.LineStyleOpts(color="grey"))
)
)
tl.add(c, "{}".format(list(data_bj['日期'])[i]))
tl.add_schema(
axis_type='time',
play_interval=100, # 表示播放的速度
pos_bottom="-29px",
is_loop_play=False, # 是否循環(huán)播放
width="780px",
pos_left='30px',
is_auto_play=True, # 是否自動播放堕汞。
is_timeline_show=False)
tl.render('1.html')
data_10 = data[(data['年份'] == 2022) & ( data['月份'] == 10)]
data_10.head()
北上廣深10月份每日最高氣溫變化
# 背景色
background_color_js = (
"new echarts.graphic.LinearGradient(0, 0, 0, 1, "
"[{offset: 0, color: '#c86589'}, {offset: 1, color: '#06a7ff'}], false)"
)
# 線條樣式
linestyle_dic = { 'normal': {
'width': 4,
'shadowColor': '#696969',
'shadowBlur': 10,
'shadowOffsetY': 10,
'shadowOffsetX': 10,
}
}
timeline = Timeline(init_opts=opts.InitOpts(bg_color=JsCode(background_color_js),
width='980px',height='600px'))
bj, gz, sh, sz= [], [], [], []
all_max = []
x_data = data_10[data_10['城市'] == '北京']['日'].tolist()
for d_time in range(len(x_data)):
bj.append(data_10[(data_10['日'] == x_data[d_time]) & (data_10['城市']=='北京')]["最高溫度"].values.tolist()[0])
gz.append(data_10[(data_10['日'] == x_data[d_time]) & (data_10['城市']=='廣州')]["最高溫度"].values.tolist()[0])
sh.append(data_10[(data_10['日'] == x_data[d_time]) & (data_10['城市']=='上海')]["最高溫度"].values.tolist()[0])
sz.append(data_10[(data_10['日'] == x_data[d_time]) & (data_10['城市']=='深圳')]["最高溫度"].values.tolist()[0])
line = (
Line(init_opts=opts.InitOpts(bg_color=JsCode(background_color_js),
width='980px',height='600px'))
.add_xaxis(
x_data,
)
.add_yaxis(
'北京',
bj,
symbol_size=5,
is_smooth=True,
is_hover_animation=True,
label_opts=opts.LabelOpts(is_show=False),
)
.add_yaxis(
'廣州',
gz,
symbol_size=5,
is_smooth=True,
is_hover_animation=True,
label_opts=opts.LabelOpts(is_show=False),
)
.add_yaxis(
'上海',
sh,
symbol_size=5,
is_smooth=True,
is_hover_animation=True,
label_opts=opts.LabelOpts(is_show=False),
)
.add_yaxis(
'深圳',
sz,
symbol_size=5,
is_smooth=True,
is_hover_animation=True,
label_opts=opts.LabelOpts(is_show=False),
)
.set_series_opts(linestyle_opts=linestyle_dic)
.set_global_opts(
title_opts=opts.TitleOpts(
title='北上廣深10月份最高氣溫變化趨勢',
pos_left='center',
pos_top='2%',
title_textstyle_opts=opts.TextStyleOpts(color='#DC143C', font_size=20)),
tooltip_opts=opts.TooltipOpts(
trigger="axis",
axis_pointer_type="cross",
background_color="rgba(245, 245, 245, 0.8)",
border_width=1,
border_color="#ccc",
textstyle_opts=opts.TextStyleOpts(color="#000"),
),
xaxis_opts=opts.AxisOpts(
# axislabel_opts=opts.LabelOpts(font_size=14, color='red'),
# axisline_opts=opts.AxisLineOpts(is_show=True,
# linestyle_opts=opts.LineStyleOpts(width=2, color='#DB7093'))
is_show = False
),
yaxis_opts=opts.AxisOpts(
name='最高氣溫',
is_scale=True,
# min_= int(min([gz[d_time],sh[d_time],sz[d_time],bj[d_time]])) - 10,
max_= int(max([gz[d_time],sh[d_time],sz[d_time],bj[d_time]])) + 10,
name_textstyle_opts=opts.TextStyleOpts(font_size=16,font_weight='bold',color='#5470c6'),
axislabel_opts=opts.LabelOpts(font_size=13,color='#5470c6'),
splitline_opts=opts.SplitLineOpts(is_show=True,
linestyle_opts=opts.LineStyleOpts(type_='dashed')),
axisline_opts=opts.AxisLineOpts(is_show=True,
linestyle_opts=opts.LineStyleOpts(width=2, color='#5470c6'))
),
legend_opts=opts.LegendOpts(is_show=True, pos_right='1%', pos_top='2%',
legend_icon='roundRect',orient = 'vertical'),
))
timeline.add(line, '{}'.format(x_data[d_time]))
timeline.add_schema(
play_interval=1000, # 輪播速度
is_timeline_show=True, # 是否顯示 timeline 組件
is_auto_play=True, # 是否自動播放
pos_left="0",
pos_right="0"
)
timeline.render('2.html')
尾語
感謝你觀看我的文章吶~本次航班到這里就結(jié)束啦 ??
希望本篇文章有對你帶來幫助 ??,有學(xué)習(xí)到一點知識~
躲起來的星星??也在努力發(fā)光晃琳,你也要努力加油(讓我們一起努力叭)讯检。