前邊已經(jīng)說到url管理器猜欺，所以這塊直接上代碼
# -*-encoding:utf-8 -*-
class UrlManager(object):
"""docstring for UrlManager"""
def __init__(self):
self.new_urls = set()#未爬取的Url集合
self.old_urls = set()#已爬取的Url集合
def has_new_url(self):
'''
判斷是否有未爬取的url
'''
return self.new_url_size()!=0
def get_new_url(self):
'''
獲取一個(gè)未爬取的url
'''
new_url = self.new_urls.pop()
self.old_urls.add(new_url)
return new_url
def add_new_url(self,url):
'''
將新的Url添加到未爬取的url集合中
'''
if url is None:
return
if url not in self.new_urls and url not in self.old_urls:
self.new_urls.add(url)
def add_new_urls(self,urls):
'''
將新的URL添加到未爬取的URL集合中
'''
if urls is None or len(urls) ==0:
return
for url in urls:
self.add_new_url(url)
def new_url_size(self):
'''
獲取未爬取的URL集合
'''
return len(self.new_urls)
def old_url_size(self):
'''
獲取已經(jīng)爬去的url集合大小
'''
return len(self.old_urls)
