python異步協(xié)程
環(huán)境:python3.7.0
協(xié)程
協(xié)程避诽，英文叫做 Coroutine稚矿，又稱微線程砸狞，纖程闷堡，協(xié)程是一種用戶態(tài)的輕量級線程。
協(xié)程本質(zhì)上是個單進(jìn)程铐伴，協(xié)程相對于多進(jìn)程來說撮奏，無需線程上下文切換的開銷，無需原子操作鎖定及同步的開銷当宴，編程模型也非常簡單畜吊。
使用協(xié)程來實現(xiàn)異步操作，發(fā)出一個請求之后户矢，需要等待一定的時間才能得到響應(yīng)玲献，這個等待過程中，程序可以干許多其他的事情，等到響應(yīng)得到之后才切換回來繼續(xù)處理捌年，這樣可以充分利用 CPU 和其他資源瓢娜，這就是異步協(xié)程的優(yōu)勢。
通過異步協(xié)程可以加快爬蟲爬取文件的速度
其中用到aiohttp庫
sudo pip3 install aiohttp
Name: aiohttp
Version: 3.4.1
Summary: Async http client/server framework (asyncio)
Home-page: https://github.com/aio-libs/aiohttp
Author: Nikolay Kim
Author-email: fafhrd91@gmail.com
License: Apache 2
Location: /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages
Requires: yarl, attrs, multidict, chardet, async-timeout
Required-by:
以下分別是用異步與不用異步爬取unsplash首頁圖片的對比
unsplash.py
import requests
import os
import re
from time import time
class Spider(object):
def __init__(self,n=10):
self.headers={
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
path='./download/unsplash'
if not os.path.exists(path):
os.mkdir(path)
self.path=path
else:
self.path=path
self.n=n
self.num=1
def getImagesLinks(self,page:int):
url='https://unsplash.com/napi/photos'
params={
'page': page,
'per_page': '12',
'order_by': 'latest'
}
links=[]
try:
r=requests.get(url=url,params=params,timeout=60)
r.raise_for_status()
r.encoding=r.apparent_encoding
for data in r.json():
links.append(data['urls']['full'])
except Exception as e:
print(e.args)
finally:
return links
def save_img(self,url):
text=""
try:
r=requests.get(url,headers=self.headers,timeout=60)
r.raise_for_status()
text=r.content
except Exception as e:
print(e.args)
finally:
return text
def download_img(self,url:str):
url_split=re.split(r'\/|\?',url)
try:
filename=url_split[3]+'.jpg'
if os.path.exists(self.path+'/'+filename):
print('下載失敗礼预，文件已存在')
else:
with open(self.path+'/'+filename,'wb') as f:
f.write(self.save_img(url))
f.close()
print('成功下載第%d張圖片'%(self.num))
self.num+=1
except Exception as e:
print(e.args)
finally:
pass
def run(self):
try:
for i in range(1,self.n+1):
urls=self.getImagesLinks(i)
for url in urls:
self.download_img(url)
except Exception as e:
print(e.args)
finally:
pass
def main():
start=time()
spider=Spider(n=1)
spider.run()
end=time()
print(end-start,'s')
if __name__ == '__main__':
main()
2.png
unsplash.py爬取里unsplash首頁的12張圖片,共耗時62.375s
async_unsplash.py
import requests
import re
import os
import asyncio
import aiohttp
from time import time
class Spider(object):
def __init__(self,n=10):
self.headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
path = './download/async_unsplash'
if not os.path.exists(path):
os.mkdir(path)
self.path = path
else:
self.path = path
self.n = n
self.num=1
def getImagesLinks(self,page:int):
url='https://unsplash.com/napi/photos'
params={
'page': page,
'per_page': '12',
'order_by': 'latest'
}
links=[]
try:
r=requests.get(url=url,params=params,timeout=60)
r.raise_for_status()
r.encoding=r.apparent_encoding
for data in r.json():
links.append(data['urls']['full'])
except Exception as e:
print(e.args)
finally:
return links
async def save_img(self,url):
content=""
try:
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
response=await session.get(url,headers=self.headers,timeout=60)
content=await response.read()
await session.close()
except Exception as e:
print(e.args)
finally:
return content
async def download_img(self,url:str):
url_split = re.split(r'\/|\?', url)
try:
filename = url_split[3] + '.jpg'
if os.path.exists(self.path + '/' + filename):
print('下載失敗眠砾，文件已存在')
else:
content=await self.save_img(url)
with open(self.path + '/' + filename, 'wb') as f:
f.write(content)
f.close()
print('成功下載第%d張圖片' % (self.num))
self.num+=1
except Exception as e:
print(e.args)
finally:
pass
def run(self):
try:
for i in range(1, self.n + 1):
urls = self.getImagesLinks(i)
tasks=[asyncio.ensure_future(self.download_img(url)) for url in urls]
loop=asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
except Exception as e:
print(e.args)
finally:
pass
def main():
start=time()
spider=Spider(n=1)
spider.run()
end=time()
print(end-start,'s')
if __name__ == '__main__':
main()
1.png
async_unsplash.py爬取里unsplash首頁的12張圖片,共耗時16.740s
通過上述兩個事例，可以很明顯看出異步協(xié)程對爬蟲爬取文件速度的提升,對別的程序運行也有同樣的作用
運行異步協(xié)程程序的時候,內(nèi)存的占有比不用時大
