簡書7日熱門文章數(shù)據(jù)分析+更新推送（持續(xù)更新···）
工作流程
- 爬取簡書7日熱門文章碎税；
- 存入MongoDB數(shù)據(jù)庫；
- 文章簡單的數(shù)據(jù)分析；
- 再次爬取簡書7日熱門文章貌矿；
- 判斷新爬取的文章是否與已爬取的內(nèi)容重復(fù)；
- 如文章不重復(fù)罪佳，發(fā)送郵件提醒逛漫；
URL結(jié)構(gòu)
首頁URL：http://www.reibang.com/trending/weekly?utm_medium=index-banner-s&utm_source=desktop
-
其他頁面URL說明：當(dāng)前頁面請求URL的seen_snote_ids位于上一頁面的
<ul class="note-list" infinite-scroll-url="/trending/weekly">的子標(biāo)簽li的data-note-id屬性中，如圖赘艳。
參考代碼
爬蟲部分代碼
import requests
from lxml import etree
homepage_url = 'http://www.reibang.com'
base_url = 'http://www.reibang.com/trending/weekly'
def get_html(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
}
resp = requests.get(url, headers=headers)
if resp.status_code == 200:
return resp.text
else:
return None
# 解析首頁數(shù)據(jù)
def parse_first_html(html, db):
root = etree.HTML(html)
articles = root.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "content", " " ))]')
seen_snote_ids = root.xpath('//ul[@class="note-list"]/li/@data-note-id')
# print(seen_snote_ids)
for article in articles:
author = ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/text()'))
author_url = homepage_url + ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/@href'))
title = ''.join(article.xpath('a[@class="title"]/text()'))
article_url = homepage_url + ''.join(article.xpath('a[@class="title"]/@href'))
abstract = ''.join(article.xpath('p[@class="abstract"]/text()')).strip()
read_and_comments = article.xpath('div[@class="meta"]/a/text()')
like_and_pay = article.xpath('div[@class="meta"]/span/text()')
# 數(shù)據(jù)處理
read_and_comments = ''.join(read_and_comments).strip().split()
like_and_pay = ''.join(like_and_pay).strip().split()
# read_count = meta.xpath('a/text()')
# print(author, author_url, title, article_url, abstract, read_and_comments, like_and_pay)
# print(''.join(like_and_pay).strip().split())
data ={
'author': author,
'author_url': author_url,
'title': title,
'article_url': article_url,
'abstract': abstract,
'read_and_comments': read_and_comments,
'like_and_pay': like_and_pay
}
if not is_data_exist(db, data):
print('保存新數(shù)據(jù)')
save_data(db, data)
else:
print('數(shù)據(jù)已存在酌毡，更新數(shù)據(jù)')
update_data(db, data)
return seen_snote_ids
def parse_other_html(html, db):
root = etree.HTML(html)
articles = root.xpath('//li/div[@class="content"]')
seen_snote_ids = root.xpath('//li/@data-note-id')
# print(seen_snote_ids)
for article in articles:
author = ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/text()'))
author_url = homepage_url + ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/@href'))
title = ''.join(article.xpath('a[@class="title"]/text()'))
article_url = homepage_url + ''.join(article.xpath('a[@class="title"]/@href'))
abstract = ''.join(article.xpath('p[@class="abstract"]/text()')).strip()
read_and_comments = article.xpath('div[@class="meta"]/a/text()')
like_and_pay = article.xpath('div[@class="meta"]/span/text()')
# 數(shù)據(jù)處理
read_and_comments = ''.join(read_and_comments).strip().split()
like_and_pay = ''.join(like_and_pay).strip().split()
# read_count = meta.xpath('a/text()')
# print(author, author_url, title, article_url, abstract, read_and_comments, like_and_pay)
data ={
'author': author,
'author_url': author_url,
'title': title,
'article_url': article_url,
'abstract': abstract,
'read_and_comments': read_and_comments,
'like_and_pay': like_and_pay
}
if not is_data_exist(db, data):
print('保存新數(shù)據(jù)')
save_data(db, data)
else:
print('數(shù)據(jù)已存在，更新數(shù)據(jù)')
update_data(db, data)
return seen_snote_ids
def get_other_html(url, page, seen_snote_ids):
url_param = url
page_param = str(page)
seen_snote_ids_param = '?seens_snote_ids%5B%5D=' + '?seens_snote_ids%5B%5D='.join(seen_snote_ids)
url = url + '?page=' + page_param + seen_snote_ids_param
# print(url)
return get_html(url)
# 初始化數(shù)據(jù)庫
def init_sql():
from pymongo import MongoClient
client = MongoClient('localhost', 27017)
# 創(chuàng)建數(shù)據(jù)庫sdifen
db = client.JIANSHU
return db
# 保存數(shù)據(jù)到數(shù)據(jù)庫中
def save_data(db, data):
db.WEEKLY.insert(data)
# 更新數(shù)據(jù)：刪除原數(shù)據(jù)蕾管，重新保存該數(shù)據(jù)枷踏？？掰曾？
def update_data(db, data):
pass
# 判斷數(shù)據(jù)庫中是否存在數(shù)據(jù)
def is_data_exist(db, data):
return db.WEEKLY.find({'article_url': data.get('article_url')}).count()
def main():
db = init_sql()
resp = get_html(base_url)
seen_snote_ids = parse_first_html(resp, db)
for i in range(2, 15):
resp = get_other_html(base_url, i, seen_snote_ids)
seen_snote_ids.extend(parse_other_html(resp, db))
if __name__ == '__main__':
main()
數(shù)據(jù)分析部分代碼
import pandas as pd
import numpy
import matplotlib.pyplot as plt
# 讀取MongoDB數(shù)據(jù)
def read_data():
import pymongo
from pymongo import MongoClient
client = MongoClient()
db = client.JIANSHU
collection = db.WEEKLY
data = pd.DataFrame(list(collection.find()))
return data
# 去重旭蠕，文章鏈接一樣的數(shù)據(jù)視為重復(fù)數(shù)據(jù)
data = read_data().drop_duplicates(['article_url'])
userClientCol = ['作者', '文章數(shù)']
# 構(gòu)造DataFrame，注意：需數(shù)組轉(zhuǎn)置
userClientDataFrame = pd.DataFrame(numpy.array([list(set(data.get('author'))), [list(data.get('author')).count(level) for level in list(set(data.get('author')))]]).T, columns=userClientCol)
# 根據(jù)作者文章數(shù)量進(jìn)行排序旷坦，排序方式為降序
userClientDataFrame = userClientDataFrame.sort_values('文章數(shù)', ascending=False)
# 選取文章數(shù)量前20名的作者
userClientDataFrame = userClientDataFrame.head(15)
plt.figure(figsize=(20,10),dpi=100)
labels = list(userClientDataFrame['作者'])
plt.bar(range(len(labels)),userClientDataFrame['文章數(shù)'],tick_label=labels)
plt.title('作者')
plt.show()
發(fā)布文章作者前20名
說明：Mac默認(rèn)的Python或者Anaconda環(huán)境下掏熬，中文會出現(xiàn)亂碼，解決方法如下：
- 下載微軟雅黑.ttf文件
- 重命名為DejaVuSans.ttf
- 替換掉/anaconda/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf（Anaconda環(huán)境）
