本文首發(fā)于我的博客：http://gongyanli.com
代碼傳送門：https://github.com/Gladysgong/BeltandRoad
簡書: http://www.reibang.com/p/6fe4afa1b98a
CSDN: https://blog.csdn.net/u012052168/article/details/79761833
好久以前做的東西了膳凝，網(wǎng)站的數(shù)據(jù)也很容易拿到讥电，最近想對自己的東西做一個總結(jié)叫惊，所以就有了這篇文章姨拥。
我的目標(biāo)是抓取一帶一路戰(zhàn)略支撐平臺里面一些機(jī)構(gòu)的數(shù)據(jù)，像聯(lián)系電話藕夫、郵箱等等昌屉，簡單看看網(wǎng)站的樣子把。
首先獲取列表項的所有url福扬，其中包括一項翻頁操作腕铸，拿到url后訪問，獲取里面的詳細(xì)信息铛碑，就這么簡單狠裹。
image
一、定義Items
`class BeltandRoadItem(scrapy.Item):
# collection = 'BeltandRoad'
name = scrapy.Field() # 機(jī)構(gòu)名稱
intro = scrapy.Field() # 機(jī)構(gòu)簡介
address = scrapy.Field() # 機(jī)構(gòu)地址
tel = scrapy.Field() # 機(jī)構(gòu)電話
fax = scrapy.Field() # 機(jī)構(gòu)傳真
email = scrapy.Field() # 機(jī)構(gòu)郵箱
site = scrapy.Field() # 機(jī)構(gòu)網(wǎng)址`
二汽烦、爬蟲模塊
爬蟲模塊中包括翻頁操作涛菠，并不復(fù)雜，就不詳細(xì)說明。
`# -*- coding:utf-8 -*-
from scrapy.spiders import CrawlSpider, Rule
import scrapy
from ..items import BeltandRoadItem
# from scrapy.conf import settings
# 一帶一路戰(zhàn)略支撐平臺
class BeltandRoadSpider(CrawlSpider):
name = "BeltandRoadSpider"
start_urls = ['http://ydyl.drcnet.com.cn/www/ydyl/channel.aspx?version=YDYL&uid=8011']
# 解析url地址
def parse(self, response):
# urls = response.xpath('//div[@class="pub_right"]/ul/li/div[1]/a/@href').extract()
urls = response.xpath('//ul[@id="ContentPlaceHolder1_WebPageDocumentsByUId1"]/li/div[1]/a/@href').extract()
# institute_name = response.xpath('//ul[@class="left-nav"]/li/h3/a/text()').extract()
for url in urls:
# url = "http://ydyl.drcnet.com.cn/www/ydyl/" + url
# print("1:", url)
yield scrapy.Request(url=url, callback=self.parse_content)
next_page = response.xpath(
'//div[@id="ContentPlaceHolder1_WebPageDocumentsByUId1_PageRow"]/input[4]/@onclick').extract()
next_page = str(next_page).split("\'")[1]
print("next:", next_page)
if next_page:
next_page = "http://ydyl.drcnet.com.cn" + next_page
yield scrapy.Request(url=next_page, callback=self.parse)
# 解析內(nèi)容
def parse_content(self, response):
item = BeltandRoadItem()
name = response.xpath('//div[@id="disArea"]/strong/div/text()').extract()[0] # 提取名稱
content = response.xpath('//div[@id="disArea"]/div[@id="docContent"]/p').xpath('string(.)').extract() # 提取其他信息
content = str(content).split('\'')
item['name'] = name
for i in range(len(content)): # 過濾無效信息后碗暗，提取有用信息存儲到item中
if content[i] != '[' and content[i] != ']' and content[i] != ', ':
print("xx:", content[i])
if "簡介" in content[i]:
item['intro'] = content[i]
elif "地址" in content[i]:
item['address'] = content[i]
elif "聯(lián)系電話" in content[i]:
item['tel'] = content[i]
elif "傳真" in content[i]:
item['fax'] = content[i]
elif "電子郵箱" in content[i]:
item['email'] = content[i]
elif "網(wǎng)址" in content[i]:
item['site'] = content[i]
yield item
`
三颈将、構(gòu)建pipelines
主要就是進(jìn)行數(shù)據(jù)持久化操作，把數(shù)據(jù)存入MongoDB數(shù)據(jù)中言疗。
`import pymongo
from scrapy.conf import settings
from .items import BeltandRoadItem
# 一帶一路戰(zhàn)略支撐平臺
class BeltandRoadPipeline(object):
def __init__(self, mongo_uri, mongo_db, mongo_port):
self.mongo_uri = mongo_uri
self.mongo_port = mongo_port
self.mongo_db = mongo_db
@classmethod
def from_crawler(cls, crawler):
return cls(mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_port=crawler.settings.get('MONGO_PORT'),
mongo_db=crawler.settings.get('MONGO_DB')
)
def open_spider(self, spider):
self.client = pymongo.MongoClient(self.mongo_uri, self.mongo_port)
self.db = self.client[self.mongo_db]
self.BeltandRoad = self.db['BeltandRoad']
def close_spider(self, spider):
self.client.close()
def process_item(self, item, spider):
if isinstance(item, BeltandRoadItem):
try:
if item['name']:
item = dict(item)
# self.db[item.collection].insert(item) 運行這個代碼時利用items中collection創(chuàng)建表晴圾，會提示插入失敗，但是依然會插入到數(shù)據(jù)庫噪奄？
self.BeltandRoad.insert(item)
print("插入成功")
return item
except Exception as e:
spider.logger.exception("插入失敗")`
四死姚、配置文件settings
# 激活pipelines
ITEM_PIPELINES = {
'BeltandRoad.pipelines.BeltandRoadPipeline': 300,}
# 數(shù)據(jù)庫配置
MONGO_URI = "127.0.0.1" # 主機(jī)IP
MONGO_PORT = 27017 # 端口號
MONGO_DB = "Belt" # 數(shù)據(jù)庫名字