簡單爬蟲實現(xiàn)，主要用到BeautifulSoup,re,urlparse, urllib2庫
項目主要結(jié)構(gòu)如下：
- crawler_main.py 項目啟動程序
- url_manager.py url管理器
- html_downloader.py html內(nèi)容下載器
- html_parser.py html解析器
- html_outputer.py html輸出器
crawler_main.py(爬蟲主程序)
import html_downloader＃html內(nèi)容下載器
import html_parser
import url_manager
import html_outputer
class SpiderMain(object):
def __init__(self):
self.urls = url_manager.UrlManager()
self.downloader = html_downloader.HtmlDownloader()
self.parser = html_parser.HtmlParser()
self.outputer = html_outputer.HtmlOutputer()
def craw(self, root_url):
count = 1
self.urls.add_new_url(root_url)＃將開始url添加到url管理中
while self.urls.has_new_url():＃查詢url管理器中是否還存在新的url
try:
new_url = self.urls.get_new_url()#從url管理器中獲取新的url
print 'craw %d : %s' %(count, new_url)
html_content = self.downloader.download(new_url)#從下載器中獲取網(wǎng)頁內(nèi)容
new_urls, new_data = self.parser.parser(new_url, html_content)＃將網(wǎng)頁內(nèi)容解析成我們想要獲取的數(shù)據(jù)，此處為新的new_urls(鏈接集合), new_data(字典數(shù)據(jù))
self.urls.add_new_urls(new_urls)#將解析后的new_urls添加到url管理器中
self.outputer.collect_data(new_data)#將解析后的內(nèi)容添加到輸出器
if count == 100:
break
count = count + 1
except BaseException, e:
print e.message
print 'craw fail'
self.outputer.output_html()#最后將內(nèi)容以網(wǎng)頁的內(nèi)容輸出
if __name__=="__main__":
root_url = "http://baike.baidu.com/item/%E8%9C%98%E8%9B%9B/8135707"
obj_crawler = SpiderMain()
obj_crawler.craw(root_url)
url_manager.py(url管理器)
# coding=utf-8
class UrlManager(object):
'''
url管理器
此處采取set(),set中不能添加相同的value
new_urls 未爬取數(shù)據(jù)的url集合
old_urls 已爬取數(shù)據(jù)的url集合
'''
def __init__(self):
self.new_urls = set()
self.old_urls = set()
def add_new_url(self, root_url):
if root_url is None:
return
#添加的url不存在new_urls中且不在old_urls中
if root_url not in self.new_urls and root_url not in self.old_urls:
self.new_urls.add(root_url)
def has_new_url(self):
return len(self.new_urls) > 0
def get_new_url(self):
new_url = self.new_urls.pop()＃從新集合中移除并獲取一條url
self.old_urls.add(new_url)
return new_url
def add_new_urls(self, new_urls):
if new_urls is None or len(new_urls) == 0:
return
for url in new_urls:
self.add_new_url(url)
html_downloader.py(html下載管理器)
import urllib2
'''從指定的url中獲取網(wǎng)頁內(nèi)容'''
class HtmlDownloader(object):
def download(self, new_url):
if new_url is None:
return None
response = urllib2.urlopen(new_url)
if response.getcode() != 200:
return None
return response.read()
html_parser.py (html解析器)
# coding=utf-8
import re
from bs4 import BeautifulSoup
import urlparse
class HtmlParser(object):
def parser(self, new_url, html_content):
if new_url is None or html_content is None:
return
soup = BeautifulSoup(html_content, 'html.parser', from_encoding='utf-8')#初始化網(wǎng)頁解析器
new_urls = self._get_new_urls(new_url, soup)#從內(nèi)容中提取爬取鏈接
new_data = self._get_new_data(new_url, soup)＃從內(nèi)容中提取想要的內(nèi)容
return new_urls, new_data
def _get_new_urls(self, new_url, soup):
new_urls = set()
links = soup.find_all('a', href=re.compile(r"/item/"))＃通過指定的正則表達式來提取符合條件的<a>標簽
for link in links:
url = link['href']#獲取href標簽內(nèi)容
new_full_url = urlparse.urljoin(new_url, url)＃通過urlparse.urljoin來拼接完整路徑的url
new_urls.add(new_full_url)
return new_urls
'''從網(wǎng)頁內(nèi)容中提取想要的內(nèi)容'''
def _get_new_data(self, new_url, soup):
res_data = {}
res_data['url'] = new_url
#< dd class ="lemmaWgt-lemmaTitle-title" > < h1 > 網(wǎng)絡(luò)爬蟲 < / h1 >
title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find("h1")#采用class_來過濾，class是關(guān)鍵詞
res_data['title'] = title_node.get_text()＃獲取提取到標簽的內(nèi)容
summary_node = soup.find('div', class_="lemma-summary")
res_data['summary'] = summary_node.get_text()
return res_data
html_outputer.py (html輸出器)
'''此處采用數(shù)組來存儲抓取到的數(shù)據(jù)'''
class HtmlOutputer(object):
def __init__(self):
self.datas = []
def collect_data(self, new_data):
if new_data is None:
return
self.datas.append(new_data)
'''自定義生成網(wǎng)頁來展示抓取到的數(shù)據(jù)'''
def output_html(self):
fout = open('output.html', 'w')
fout.write('<html>')
fout.write('<head>')
fout.write("<meta charset='utf-8'>")
fout.write('</head>')
fout.write('<body>')
fout.write('<table border="1">')
for data in self.datas:
fout.write('<tr>')
fout.write("<td width='200px'>%s</td>" % data['url'])
fout.write("<td width='100px'>%s</td>" % data['title'].encode('utf-8'))#由于存在中文智厌，需要指定編碼格式
fout.write("<td>%s</td>" % data['summary'].encode('utf-8'))
fout.write('</tr>')
fout.write('</table>')
fout.write('</body>')
fout.write('</html>')
fout.close()
