365yg視頻抓取--def方法
得到網(wǎng)址逊拍，但這個(gè)視頻接口無(wú)法用普通方法下載，現(xiàn)存入本地际邻，若以后有可下載軟件芯丧，即可使用
import requests
from lxml import etree
import json
from selenium import webdriver
from time import sleep
#爬取365yg的視頻
#相關(guān)網(wǎng)址"http://www.365yg.com/"
#定義一個(gè)函數(shù)處理json數(shù)據(jù)
def handle_json(json_url, page):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}
r = requests.get(url=json_url%page, headers=headers)
#對(duì)json數(shù)據(jù)進(jìn)行解析,生成obj類型
obj = json.loads(r.text)
return obj["data"]
#定義一個(gè)函數(shù)，用于處理json數(shù)據(jù)
def handle_video(video_list):
for video in video_list:
#獲取vedio_id作為鍵
video_id = video["video_id"]
source_url = "http://365yg.com" + video["source_url"]
download_video(source_url, video_id)
def download_video(video_url, video_id):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}
# v_data = requests.get(url=video_url, headers=headers)
# with open("video.html", "wb") as fp:
# fp.write(v_data.content)
#通過(guò)頁(yè)面下載發(fā)現(xiàn)該頁(yè)面是動(dòng)態(tài)加載
driver = webdriver.Chrome()#PhantomJS不支持此頁(yè)面可能版本太舊世曾，作者已不更新PhantomJS
driver.get(video_url)
sleep(3)
#用xpath選取video的url
html_tree = etree.HTML(driver.page_source)
video_src = html_tree.xpath("http://video/@src")[0]
# print(video_src)
sleep(3)
driver.quit()
#得到網(wǎng)址缨恒，但這個(gè)視頻接口無(wú)法用普通方法下載，現(xiàn)存入本地轮听，若以后有可下載軟件骗露，即可使用
res = requests.get(video_src, headers=headers)
# with open("video1.html", "w") as fp:
# fp.write(res)
def main():
start = int(input("請(qǐng)輸入起始頁(yè)："))
end = int(input("請(qǐng)輸入終止頁(yè)："))
json_url = "https://365yg.com/api/pc/feed/?category=video&utm_source=toutiao&widen=%d&max_behot_time=0&max_behot_time_tmp=0&tadrequire=true&as=A1253AF48A0FCE5&cp=5A4A4F4CAEE59E1&_signature="
# 定義一個(gè)列表變量，用于整合所有的json數(shù)據(jù)
video_list = []
for i in range(int(start), int(end)+1):
json_list = handle_json(json_url, i)
video_list += json_list
return video_list
if __name__ == "__main__":
main()
