這個(gè)系列主要是對(duì)pandas庫的熟悉扒秸，這篇筆記是對(duì)大鵬老師課程的學(xué)習(xí)播演，主要是對(duì)去哪兒網(wǎng)數(shù)據(jù)的爬取冀瓦，分析城市的熱門景點(diǎn)。

1写烤、數(shù)據(jù)爬取

# 導(dǎo)入工具包
import requests
from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# 獲取urls(更具url的規(guī)律)
def get_urls(ui,n):
    urllsti = []
    for i in range(1,n+1):
        urllsti.append(ui +str(i))
    return urllsti

# 批量獲取數(shù)據(jù)
def get_data(u):
    ri = requests.get(u)
    soupi = BeautifulSoup(ri.text,'lxml')
    infori = soupi.find('ul',class_="list_item clrfix").find_all('li')
      
    datai = []
    n=0
    for i in infori:
        n+=1
        dic = {}
        dic['lat'] = i['data-lat']
        dic['lng'] = i['data-lng']
        dic['景點(diǎn)名稱'] = i.find('span',class_="cn_tit").text
        dic['攻略提到數(shù)量'] = i.find('div',class_="strategy_sum").text
        dic['點(diǎn)評(píng)數(shù)量'] = i.find('div',class_="comment_sum").text
        dic['景點(diǎn)排名'] = i.find('span',class_="ranking_sum").text
        dic['星級(jí)'] = i.find('span',class_="total_star").find('span')['style'].split(':')[1]
        datai.append(dic)
    return datai

# 獲取40條北京的景點(diǎn)數(shù)據(jù)
bj_u = 'https://travel.qunar.com/p-cs299914-beijing-jingdian-1-'
    # 頁面參數(shù)
urls = get_urls(bj_u,4)
    # 獲取所有url
bj_data = []
for i in urls:
    bj_data.extend(get_data(i))
    print('成功采集%i條數(shù)據(jù)' % len(bj_data))
    # 采集數(shù)據(jù)

df = pd.DataFrame(bj_data)
df.head()

image.png

2翼闽、數(shù)據(jù)清洗

# 字段類型處理
df.index = df['景點(diǎn)名稱']
del df['景點(diǎn)名稱']
df['lng'] = df['lng'].astype(np.float)
df['lat'] = df['lat'].astype(np.float)    
df['點(diǎn)評(píng)數(shù)量'] = df['點(diǎn)評(píng)數(shù)量'].astype(np.int)     
df['攻略提到數(shù)量'] = df['攻略提到數(shù)量'].astype(np.int)   
df.head()

# 星級(jí)字段處理
df['星級(jí)'] = df['星級(jí)'].str.replace('%','').astype(np.float)
df.head()

# 景點(diǎn)排名處理
df['景點(diǎn)排名'] = df['景點(diǎn)排名'].str.split('第').str[1]
df['景點(diǎn)排名'].fillna(value = 0,inplace = True) 
df.head()

3、數(shù)據(jù)查看

# 查看點(diǎn)評(píng)數(shù)量TOP10
dptop10 = df.sort_values(by = '點(diǎn)評(píng)數(shù)量', ascending=False).iloc[:10]
dptop10['點(diǎn)評(píng)數(shù)量'].plot(kind='bar',figsize = (10,5),rot=45,grid=True,color='y')

# 攻略提到數(shù)量排名TOP10
gltop10 = df.sort_values(by = '攻略提到數(shù)量', ascending=False).iloc[:10]
gltop10['攻略提到數(shù)量'].plot(kind='bar',figsize = (10,5),rot=45,grid=True,color='g')

4洲炊、景點(diǎn)篩選機(jī)制及評(píng)價(jià)方法

# 滿意度指標(biāo)
df['滿意度'] = df['攻略提到數(shù)量']/df['點(diǎn)評(píng)數(shù)量']
df.head()

# 構(gòu)建函數(shù)實(shí)現(xiàn)字段標(biāo)準(zhǔn)化
def nordata(dfi,*cols):
    for col in cols:
        dfi[col + '_nor'] = (dfi[col] - dfi[col].min())/(dfi[col].max() - dfi[col].min())
 
nordata(df,'滿意度','星級(jí)','點(diǎn)評(píng)數(shù)量')
df.head()

image.png