- 在settings中激活管道檬某、設(shè)置數(shù)據(jù)庫參數(shù)
- 導(dǎo)入twisted的異步插入數(shù)據(jù)庫的模塊
from twisted.enterprise import adbapi
from chufang.items import ChufangUserItem, ChufangMenuItem, ChufangTagsItem
import scrapy, pymysql
class ChufangPipeline(object):
def __init__(self, dbpool):
self.dbpool = dbpool
@classmethod
def from_crawler(cls, cralwer):
db_parmars = {
'host': cralwer.settings['MYSQL_HOST'],
'user': cralwer.settings['MYSQL_USER'],
'passwd': cralwer.settings['MYSQL_PWD'],
'db': cralwer.settings['MYSQL_DB'],
'port': cralwer.settings['MYSQL_PORT'],
'charset': cralwer.settings['MYSQL_CHARSET']
}
dbpool = adbapi.ConnectionPool('pymysql', **db_parmars)
return cls(dbpool)
def process_item(self, item, spider):
query = self.dbpool.runInteraction(
self.insert_data_to_mysql,
item
)
query.addErrback(
self.insert_err,
item
)
return item
def insert_data_to_mysql(self, cursor, item):
data_dict = dict(item)
sql, data = item.get_insert_sql_data(data_dict)
cursor.execute(sql, data)
def insert_err(self, failure, item):
print(failure, '失敗', item)
最后編輯于 ：
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請(qǐng)聯(lián)系作者
