一袭景、pyspider簡介
參考文檔：
二候址、需要安裝的依賴包
sudo apt-get install python
sudo apt-get install python-dev
sudo apt-get install python-distribute
sudo apt-get install python-pip
sudo apt-get install libcurl4-openssl-dev
sudo apt-get install libxml2-dev
sudo apt-get install libxslt1-dev
sudo apt-get install python-lxml libssl-dev
sudo apt-get install zlib1g-dev
sudo apt-get install phantomjs
pip3 install pyspider
啟動(dòng)
pyspider all
代碼樣例
from pyspider.libs.base_handler import *
import pymongo,pymysql
class Handler(BaseHandler):
"""
Handler 就是 pyspider 爬蟲的主類，我
們可以在此處定義爬取楞慈、解析璧坟、存儲(chǔ)的邏輯嗡贺。
整個(gè)爬蟲的功能只需要一個(gè) Handler 即可完成
"""
#crawl_config 屬性。我們可以將本項(xiàng)目的
#所有爬取配置統(tǒng)一定義到這里抖拦，如定義
#Headers升酣、設(shè)置代理等，配置之后全局生效
crawl_config = {
}
#mongodb數(shù)據(jù)庫連接
mongocli = pymongo.MongoClient(host='localhost', port=27017)
db = mongocli['jobbole']
jobbolearticle = db['jobbolearticle']
#mysql數(shù)據(jù)庫連接
client = pymysql.Connect(
'localhost','root','xxx',
'xxx',charset='utf8'
)
cursor = client.cursor()
#on_start() 方法是爬取入口态罪，初始的爬取
#請求會(huì)在這里產(chǎn)生
@every(minutes=24 * 60)
def on_start(self):
self.crawl('http://blog.jobbole.com/all-posts/', callback=self.index_page)
@config(age=10 * 24 * 60 * 60)
def index_page(self, response):
print(response)
for each in response.doc('a[class="page-numbers"]').items():
self.crawl(each.attr.href, callback=self.detail_page)
@config(priority=2)
def detail_page(self, response):
self.index_page(response)
list = response.doc('#archive .post.floated-thumb')
# print(list)
for item in list.items():
print('拿到了數(shù)據(jù)')
print(type(item))
title = item('a.archive-title').text()
url = item('a.archive-title').attr.href
print(title,url)
return {
'title':title,
'url':url,
}
#方法中return的結(jié)果會(huì)執(zhí)行on_result
def on_result(self,result):
#可以在這里做數(shù)據(jù)的持久化
print(result)
#mysql數(shù)據(jù)庫存儲(chǔ)
sql = """
INSERT INTO jobbole()
VALUE (%s,%s)
"""
try:
self.cursor.execute(sql,[result['title'],result['url']])
self.client.commit()
except Exception as err:
print(err)
self.client.rollback()
#mongodb數(shù)據(jù)庫存儲(chǔ)
self.jobbolearticle.insert(result)
