- 需要觀察房天下url的構(gòu)造忍些，本次爬取的是新房和二手房兩個欄目的具體字段。
- 涉及到的知識點(diǎn)有url的拼接轰异，具體字段的解析清洗岖沛，頁面不規(guī)整的情況下，怎樣提取搭独。
- 分布式部署的相關(guān)操作
- 在爬的時候建議網(wǎng)頁延遲多一些婴削。
fangtianxia.py文件
import scrapy,re
from fang.items import NewHouseItem,ESFHouseItem
class FangtianxiaSpider(scrapy.Spider):
name = 'fangtianxia'
allowed_domains = ['fang.com']
start_urls = ['http://www.fang.com/SoufunFamily.htm']
def parse(self, response):
trs = response.xpath('//div[@class="outCont"]//tr')
province = None # 首先設(shè)為沒有值，下方判斷有值在賦給province
for tr in trs:
tds = tr.xpath('.//td[not(@class)]')
province_td = tds[0] # 提取省份牙肝，由于省份不是每一行都有的唉俗，所以要過濾一下
province_text = province_td.xpath('.//text()').get() # 沒有省份的那一行會有空格
province_text = re.sub(r'\s','',province_text) # 用sub替換一下，好判斷
if province_text:
province = province_text # 如果有值配椭，就賦給province
if '其它' in province: # 不提取海外的
continue
city_id = tds[1] # 接下來提取城市鏈接和城市名稱
city_links = city_id.xpath('.//a')
for city_link in city_links:
city_url = city_link.xpath('.//@href').get()
city = city_link.xpath('.//text()').get()
# 構(gòu)建新房和二手房的url
url_module = city_url.split('fang')
prefix = url_module[0]
domain = url_module[1]
# 北京特殊虫溜，特殊處理一下
if 'bj' in prefix:
newhouse_url = 'http://' + 'newhouse.fang' + domain + 'house/s/'
esf_url = 'http://' + 'esf.fang' + domain
else:
# 構(gòu)建新房的url
newhouse_url = prefix + 'newhouse.fang' + domain + 'house/s/'
# 構(gòu)建二手房的url
esf_url = prefix + 'esf.fang' + domain
# meta里面可以攜帶一些參數(shù)信息放到Request里面，在callback函數(shù)里面通過response獲取
yield scrapy.Request(url=newhouse_url,callback=self.parse_newhouse,meta={'info':(province,city)})
yield scrapy.Request(url=esf_url,callback=self.parse_esf,meta={'info':(province,city)})
def parse_newhouse(self,response):
# 解析新房具體字段
# meta里面可以攜帶一些參數(shù)信息放到Request里面股缸，在callback函數(shù)里面通過response獲取
province,city = response.meta.get('info')
lis = response.xpath('//div[contains(@class,"nl_con")]/ul/li')
for li in lis:
name = li.xpath(".//div[contains(@class,'house_value')]//div[@class='nlcd_name']/a/text()").get()
if name:
name = re.sub(r"\s","",name)
house_type_list = li.xpath('.//div[contains(@class,"house_type")]/a/text()').getall()
#house_type_list = list(map(lambda x:x.replace(' ',''),house_type_list))
house_type_list = list(map(lambda x:re.sub(r'/s','',x),house_type_list))
rooms = list(filter(lambda x:x.endswith('居'),house_type_list))
area = ''.join(li.xpath('.//div[contains(@class,"house_type")]/text()').getall())
area = re.sub(r'\s|－|/','',area)
address = li.xpath('.//div[@class="address"]/a/@title').get()
# district_text = ''.join(li.xpath('.//div[@class="address"]/a//text()').getall())
# district = re.search(r'.*\[(.+)\].*',district_text).group(1)
sale = li.xpath(".//div[contains(@class,'fangyuan')]/span/text()").get()
price = "".join(li.xpath(".//div[@class='nhouse_price']//text()").getall())
price = re.sub(r"\s|廣告", "", price)
# 詳情頁url
origin_url = li.xpath(".//div[@class='nlcd_name']/a/@href").get()
item = NewHouseItem(name=name,rooms=rooms.get(),area=area,address=address,
sale=sale,price=price,origin_url=origin_url,province=province,city=city)
yield item
# 下一頁
# next_url = response.xpath("http://div[@class='page']//a[@class='next']/@href").get()
# if next_url:
# yield scrapy.Request(url=response.urljoin(next_url),
# callback=self.parse_newhouse,
# meta={'info': (provice, city)}
# )
def parse_esf(self, response):
# 二手房
province, city = response.meta.get('info')
dls = response.xpath("http://div[@class='shop_list shop_list_4']/dl")
for dl in dls:
item = ESFHouseItem(province=province,city=city)
name = dl.xpath(".//span[@class='tit_shop']/text()").get()
if name:
infos = dl.xpath(".//p[@class='tel_shop']/text()").getall()
infos = list(map(lambda x: re.sub(r"\s", "", x), infos))
for info in infos:
if "廳" in info:
item["rooms"] = info
elif '層' in info:
item["floor"] = info
elif '向' in info:
item['toward'] = info
elif '㎡' in info:
item['area'] = info
elif '年建' in info:
item['year'] = re.sub("年建", "", info)
item['address'] = dl.xpath(".//p[@class='add_shop']/span/text()").get()
# 總價
item['price'] = "".join(dl.xpath(".//span[@class='red']//text()").getall())
# 單價
item['unit'] = dl.xpath(".//dd[@class='price_right']/span[2]/text()").get()
item['name'] = name
detail = dl.xpath(".//h4[@class='clearfix']/a/@href").get()
item['origin_url'] = response.urljoin(detail)
yield item
# 下一頁
# next_url = response.xpath("http://div[@class='page_al']/p/a/@href").get()
# if next_url:
# yield scrapy.Request(url=response.urljoin(next_url),
# callback=self.parse_esf,
# meta={'info': (provice, city)}
# )
item.py文件
import scrapy
from scrapy import Field
class NewHouseItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
# 省份
province = Field()
# 城市
city = Field()
# 小區(qū)名字
name = Field()
# 價格
price = Field()
# 幾居室衡楞，這是一個列表
rooms = Field()
# 面積
area = Field()
# 地址
address = Field()
sale = Field()
# 房天下詳情url
origin_url = Field()
class ESFHouseItem(scrapy.Item):
# 省份
province = Field()
# 城市
city = Field()
# 小區(qū)名字
name = Field()
# 幾室?guī)讖d
rooms = Field()
# 層
floor = Field()
# 朝向
toward = Field()
# 年代
year = Field()
# 地址
address = Field()
# 建筑面積
area = Field()
# 總價
price = Field()
# 單價
unit = Field()
# 詳情頁url
origin_url = Field()
settings.py文件
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 1
from fake_useragent import UserAgent
ua = UserAgent().random
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'User-Agent':ua
}
ITEM_PIPELINES = {
'fang.pipelines.FangPipeline': 300,
'fang.pipelines.MongoPipeline': 400,
}
MONGO_URI = 'localhost'
MONGO_DB = 'fangtianxia'
pipelines.py文件
from scrapy.exporters import JsonLinesItemExporter
class FangPipeline(object):
def __init__(self):
self.newhouse_fp = open('newhouse.json','wb')
self.esfhouse_fp = open('esfhouse.json','wb')
self.newhouse_exporter = JsonLinesItemExporter(self.newhouse_fp,ensure_ascii=False)
self.esfhouse_exporter = JsonLinesItemExporter(self.esfhouse_fp,ensure_ascii=False)
def process_item(self, item, spider):
self.newhouse_exporter.export_item(item)
self.esfhouse_exporter.export_item(item)
return item
def close_spider(self,spider):
self.newhouse_fp.close()
self.esfhouse_fp.close()
# 保存到mongodb
class MongoPipeline(object):
def __init__(self,mongo_uri,mongo_db):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
@classmethod
def from_crawler(cls,crawler):
return cls(
mongo_uri = crawler.settings.get('MONGO_URI'),
mongo_db = crawler.settings.get('MONGO_DB')
)
def open_spider(self,spider):
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
def process_item(self,item,spider):
name = item.__class__.__name__
self.db[name].insert(dict(item))
return item
def close_spider(self,spider):
self.client.close()
改造成分布式爬蟲
首先安裝pip install scrapy-redis
要將一個Scrapy項(xiàng)目變成一個Scrapy-redis項(xiàng)目只需修改以下三點(diǎn)就可以了：
- 將爬蟲的類從 scrapy.Spider 變成 scrapy_redis.spiders.RedisSpider；或者是從 scrapy.CrawlSpider 變成 scrapy_redis.spiders.RedisCrawlSpider敦姻。
拿上面的例子來說就是在 fangtianxia.py文件中
from scrapy_redis.spiders import RedisSpider
class FangtianxiaSpider(RedisSpider):
name = 'fangtianxia'
allowed_domains = ['fang.com']
# start_urls = ['http://www.fang.com/SoufunFamily.htm']
redis_key = "fang:start_urls"
- 將爬蟲中的start_urls刪掉瘾境。增加一個redis_key="xxx"歧杏。這個redis_key是為了以后在redis中控制爬蟲啟動的。爬蟲的第一個url迷守，就是在redis中通過這個發(fā)送出去的犬绒。
- 更改scrapy的調(diào)度器，用redis實(shí)現(xiàn)的調(diào)度器兑凿。url去重的工作也交由redis完成凯力，爬取的數(shù)據(jù)共享一下，存儲到redis礼华。在配置文件中增加如下配置：
# Scrapy-Redis相關(guān)配置
# 確保request存儲到redis中
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 確保所有爬蟲共享相同的去重指紋
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 設(shè)置redis為item pipeline
ITEM_PIPELINES = {
'scrapy_redis.pipelines.RedisPipeline': 300
}
# 在redis中保持scrapy-redis用到的隊列咐鹤，不會清理redis中的隊列，從而可以實(shí)現(xiàn)暫停和恢復(fù)的功能卓嫂。
SCHEDULER_PERSIST = True
# 設(shè)置連接redis信息
REDIS_HOST = '127.0.0.1'
REDIS_PORT = 6379
運(yùn)行爬蟲：
- 在爬蟲服務(wù)器上慷暂。進(jìn)入爬蟲文件所在的路徑聘殖，然后輸入命令：scrapy runspider [爬蟲名字]晨雳。
- 在Redis服務(wù)器上，推入一個開始的url鏈接：redis-cli> lpush [redis_key] start_url開始爬取奸腺。
