[TOC]
功能點(diǎn)
關(guān)鍵代碼簡要說明
完整代碼
文中使用的python請求數(shù)據(jù)地址來源于 ruanyf/sina-news
python3
功能點(diǎn)
抓取sina的新聞列表
將抓取到的數(shù)據(jù)保存成一個(gè)json格式的json文件
關(guān)鍵代碼簡要說明
-
裝載容器
將每條新聞封裝為一個(gè)Model類
class Model:
def __init__(self, id, title, updateTime, jumpLink, tags) :
self.id = id # 每條新聞的id
self.title = title # 每條新聞的title
self.updateTime = updateTime # 每條新聞的發(fā)布/更新時(shí)間
self.jumpLink = jumpLink # 每條新聞的詳情
self.tags = tags # 每條新聞的tag列表</pre>
-
對數(shù)據(jù)的操作
- 打開文件/保存
def feedaskByPage(page): # 分頁抓取和保存
with open(jsonFileName, "a+", encoding='UTF-8') as file:
#...#
# 如果page != 1隐砸， 則說明不是第一頁之碗，則需要先讀取原本保存的json文件后，將數(shù)據(jù)拼接到原數(shù)據(jù)后面
if page != 1:
# 因?yàn)榇蜷_文件的方式是 a+, 所以文件的讀取指針是文件的最后一位季希，需要將指針移到第一位
file.seek(0, 0)
toFileList = JSON.loads(file.read())
#...#
# 清空原本的文件
file.seek(0)
file.truncate()
# print(toFileList.__len__())
# 在保有json格式下保存到文件中
_dataStr = JSON.dumps(toFileList, ensure_ascii = False, indent = 4) # indent = 4 為json的行空格褪那，這樣保存的文件會(huì)以相對比較容易看看懂的形式存在，非必要參數(shù)
file.write(_dataStr)
file.close()
- 分頁請求
import time
# 部分請求參數(shù)
reqParams = {
'page': 1,
'page_size': 100
#...#
}
#...#
response = requests.get(url=reqUrl, params= reqParams, headers= reqHeaders)
reqParams['page'] = page
json = response.json()
#...#
def feedtask():
print("start new round ....")
for index in range(1, reqPage + 1):
# print(index)
feedaskByPage(index)
time.sleep(2) # 分頁的請求間隔時(shí)間
print("finish this round ....")
- 定時(shí)任務(wù)
import schedule
import time
#...#
schedule.every(15).minutes.do(feedtask) # 每15分鐘抓取一次
print("start schedule task ....")
while True:
schedule.run_pending() # 運(yùn)行所有可以運(yùn)行的任務(wù)
time.sleep(1)
完整代碼
python3
import requests
import json as JSON
import schedule
import time
reqUrl = 'https://zhibo.sina.com.cn/api/zhibo/feed'
reqParams = {
'page': 1,
'page_size': 100,
'zhibo_id': 152, ## 新聞專用
'tag_id': 0,
'dire': 'f',
'dpc': 1,
'type': 0
}
reqHeaders = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10130'}
reqPage = 1 ## TODO 需要處理時(shí)間的排序問題
jsonFileName = "./your_json_file_name.json"
class ToFileModel:
def __init__(self, id, title, updateTime, jumpLink, tags) :
self.id = id
self.title = title
self.updateTime = updateTime
self.title = title
self.jumpLink = jumpLink
self.tags = tags
def feedaskByPage(page):
with open(jsonFileName, "a+", encoding='UTF-8') as file:
response = requests.get(url=reqUrl, params= reqParams, headers= reqHeaders)
reqParams['page'] = page
json = response.json()
result = json['result']
data = result['data']
feed = data['feed']
list = feed['list']
toFileList = []
toTagList = []
# _data = {}
if page == 1:
file.seek(0)
file.truncate()
else :
file.seek(0, 0)
# _data = JSON.loads(file.read())
# toFileList = _data['list']
toFileList = JSON.loads(file.read())
for tmp in list:
for tag in tmp['tag']:
toFileTag = tag['name']
toTagList.append(toFileTag)
toFileList.append(
ToFileModel(tmp['id'], tmp['rich_text'], tmp['update_time'], tmp['docurl'], toTagList)
.__dict__)
toTagList = []
file.seek(0)
file.truncate()
print(toFileList.__len__())
# _data['list'] = toFileList
# JSON.dump(_data, file, ensure_ascii = False, indent = 4)
_dataStr = JSON.dumps(toFileList, ensure_ascii = False, indent = 4)
file.write(_dataStr)
file.close()
def feedtask():
print("start new round ....")
for index in range(1, reqPage + 1):
# print(index)
feedaskByPage(index)
time.sleep(2)
print("finish this round ....")
# schedule.every(3).seconds.do(feedtask)
schedule.every(15).minutes.do(feedtask)
print("start schedule task ....")
while True:
schedule.run_pending() # 運(yùn)行所有可以運(yùn)行的任務(wù)
time.sleep(1)
