import threading
import time
from lxmlimport etree
import requests
from queueimport Queue
import re
"""
其實(shí) 函數(shù)參數(shù)中的冒號(hào)是參數(shù)的類型建議符，告訴程序員希望傳入的實(shí)參的類型 。 函數(shù)后面跟著的箭頭是函數(shù)返回值的類型建議符
骗爆，用來(lái)說(shuō)明該函數(shù)返回的值是什么類型 。 更官方的解釋：此為type hints须鼎，是Python 3.5新加的"""
headers = {
'user-agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55",
? ? 'refer':'https://so.toutiao.com/search?keyword=%E8%A1%97%E6%8B%8D%E7%BE%8E%E5%A5%B3&pd=information&source=pagination&dvpf=pc&aid=4916&page_num={page}&search_id=202203201951110102120460753D61F421'
}
cookie={'cookie':'tt_webid=7077014656070960676; _S_DPR=1.5; _S_IPAD=0; MONITOR_WEB_ID=7077014656070960676; FRM=new; PIXIEL_RATIO=1.5; WIN_WH=1280_648; ttcid=c9720ff1a7c146a790a1c113c8f8e91f39; tt_scid=0LEZtCnx3EI6WHPLeDb9vGDK-P-YwQDee.cVi7b6xfkeANJ9dxJUAUIHgJiGKGF08d18; ttwid=1%7C5pSenh2ZSIAoAPAnZmcHQ2xN3TykA8ikp6iCucnETC0%7C1648709296%7Ce85f11b922972468dc1718583be6152a42d5113842d03386a442d9afccd8b8aa; _S_WIN_WH=1280_648'}
class Producer(threading.Thread):
def __init__(self, page_queue,img_queue, *args, **kwargs):
super(Producer, self).__init__(*args, **kwargs)
self.page_queue= page_queue
self.img_queue = img_queue
def run(self):
while True:
if self.page_queue.empty():
break
? ? ? ? ? ? else:
url =self.page_queue.get()
self.parse_page(url)
def parse_page(self, url):
response = requests.get(url=url, headers=headers,cookies=cookie)
text = response.text
html = etree.HTML(text)
img_list = html.xpath('//div[@class="cs-view cs-view-block cs-card-content"]')
for liin img_list:
href = li.xpath('./a/@href')
if href:
m = re.findall(r'.*(\d{19}).*', href[0])
a ='https://www.toutiao.com/a' + m[0] +'/?channel=&source=search_tab'
? ? ? ? ? ? ? ? self.parser2(a)
def parser2(self,a):
response = requests.get(url=a, headers=headers,cookies=cookie )
m = etree.HTML(response.text)
s = m.xpath('//article[@class="syl-article-base tt-article-content syl-page-article syl-device-pc"]')[0]
x = s.xpath('./div[@class="pgc-img"]')
for iin x:
ll = i.xpath('./img/@src')
if ll:
self.img_queue.put(ll[0])
class Consumer(threading.Thread):
def __init__(self, page_queue, img_queue, *args, **kwargs):
super(Consumer, self).__init__(*args, **kwargs)
self.page_queue = page_queue
self.img_queue = img_queue
print('af',self.img_queue.get())
def run(self):
while True:
if self.page_queue.empty()and self.img_queue.empty():
break
? ? ? ? ? ? else:
imgurl=self.img_queue.get()
name = imgurl[-11:-8]
path =r"C:\Users\29258\Desktop\er\\" + name +'.jpg'
? ? ? ? ? ? ? ? response = requests.get(url=imgurl, headers=headers,)
with open(path, 'wb')as fp:
fp.write(response.content)
print(name,'下載好了')
def main():
page_queue = Queue(50)# 存儲(chǔ)頁(yè)碼鏈接
? ? img_queue = Queue(10)# 存儲(chǔ)解析出來(lái)的圖片鏈接
? ? # 想要爬取前10頁(yè)的數(shù)據(jù)
? ? for pagein range(50):
url =f'https://so.toutiao.com/search?keyword=%E8%A1%97%E6%8B%8D%E7%BE%8E%E5%A5%B3&pd=information&source=pagination&dvpf=pc&aid=4916&page_num={page}&search_id=202203201951110102120460753D61F421'
? ? ? ? print(url)
page_queue.put(url)# 將10頁(yè)的頁(yè)碼鏈接加入到了page_queue
? ? for xin range(3):
t = Producer(page_queue, img_queue)
t.start()
# 開(kāi)啟三個(gè)消費(fèi)者
? ? for xin range(5):
t = Consumer(page_queue, img_queue)
t.start()
time.sleep(3)
if __name__=='__main__':
main()
# 開(kāi)啟三個(gè)生產(chǎn)者
? ? # for x in range(3):
#? ? t = Producer(page_queue, img_queue)
#? ? t.start()
? # # 開(kāi)啟三個(gè)消費(fèi)者
? ? # for x in range(3):
#? ? t = Consumer(page_queue, img_queue)
#? ? t.start()
原創(chuàng)
