目的
Scrapy框架為文件和圖片的下載專門提供了兩個Item Pipeline 它們分別是：
FilePipeline
ImagesPipeline
這里主要介紹ImagesPipelineＡ甘苍！
目標(biāo)分析：
這次我們要爬的是 汽車之家：car.autohome.com.cn
最近喜歡吉利博越步脓，所以看了不少這款車的資料鸠姨。。。。
我們就點(diǎn)開博越汽車的圖片網(wǎng)站：
https://car.autohome.com.cn/pic/series/3788.html
傳統(tǒng)的Scrapy框架圖片下載
Scrapy 框架的實(shí)施：
-
創(chuàng)建scrapy項(xiàng)目和爬蟲：
$ scrapy startproject Geely $ cd Geely $ scrapy genspider BoYue car.autohome.com.cn -
編寫items.py:
import scrapy class GeelyItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() # 存儲圖片分類 catagory = scrapy.Field() # 存儲圖片地址 image_urls = scrapy.Field() # ImagesPipeline images = scrapy.Field() -
編寫Spider:
獲取高清圖片：
通過分析縮略圖和高清圖的url，我們發(fā)現(xiàn)縮略圖只是多了t_罷了
縮略圖地址：
高清圖地址：
https://car3.autoimg.cn/cardfs/product/g25/M09/B0/61/autohomecar__wKgHIlpxMgyAWYh2AAaCZ2odx24585.jpg
# -*- coding: utf-8 -*- import scrapy #導(dǎo)入CrawlSpider模塊 需改寫原來的def parse(self,response)方法 from scrapy.spiders import CrawlSpider ,Rule #導(dǎo)入鏈接提取模塊 from scrapy.linkextractors import LinkExtractor from Geely.items import GeelyItem class BoyueSpider(CrawlSpider): name = 'BoYue' allowed_domains = ['car.autohome.com.cn'] start_urls = ['https://car.autohome.com.cn/pic/series/3788.html'] #如需要進(jìn)行頁面解釋則使用callback回調(diào)函數(shù) 因?yàn)橛邢乱豁摿虢樱晕覀冃枰M(jìn)，這里使用follow令其為True rules = { Rule(LinkExtractor(allow=r'https://car.autohome.com.cn/pic/series/3788.+'), callback= 'parse_page', follow=True), } def parse_page(self, response): catagory = response.xpath('//div[@class = "uibox"]/div/text()').get() srcs = response.xpath('//div[contains(@class,"uibox-con")]/ul/li//img/@src').getall() #map(函數(shù)臼予，參數(shù)二)鸣戴，將參數(shù)二中的每個都進(jìn)行函數(shù)計(jì)算并返回一個列表 srcs = list(map(lambda x:x.replace('t_',''),srcs)) srcs = list(map(lambda x:response.urljoin(x),srcs)) yield GeelyItem(catagory=catagory, image_urls = srcs) -
編寫PIPELINE：
import os from urllib import request class GeelyPipeline(object): def __init__(self): #os.path.dirname()獲取當(dāng)前文件的路徑,os.path.join()獲取當(dāng)前目錄并拼接成新目錄 self.path = os.path.join(os.path.dirname(__file__), 'images') # 判斷路徑是否存在 if not os.path.exists(self.path): os.mkdir(self.path) def process_item(self, item, spider): #分類存儲 catagory = item['catagory'] urls = item['image_urls'] catagory_path = os.path.join(self.path, catagory) #如果沒有該路徑即創(chuàng)建一個 if not os.path.exists(catagory_path): os.mkdir(catagory_path) for url in urls: #以_進(jìn)行切割并取最后一個單元 image_name = url.split('_')[-1] request.urlretrieve(url,os.path.join(catagory_path,image_name)) return item -
編寫settings.py
BOT_NAME = 'Geely' SPIDER_MODULES = ['Geely.spiders'] NEWSPIDER_MODULE = 'Geely.spiders' # Obey robots.txt rules ROBOTSTXT_OBEY = False ITEM_PIPELINES = { 'Geely.pipelines.GeelyPipeline': 1, } -
讓項(xiàng)目跑起來：
$ scrapy crawl BoYue -
結(jié)果展示:
使用Images_pipeline進(jìn)行圖片下載
使用步驟：
定義好一個item,然后定義兩個屬性 image_urls 和 images。 image_urls是用來存儲需要下載的文件的url鏈接粘拾，列表類型窄锅；
當(dāng)文件下載完成后，會把文件下載的相關(guān)信息存儲到item的images屬性中。例如：下載路徑入偷，下載url 和文件的效驗(yàn)碼追驴；
再配置文件settings.py中配置FILES_STORE,指定文件下載路徑；
啟動pipeline,在ITEM_PIPELINES中設(shè)置自定義的中間件Ｊ柚５钛！
具體步驟
在上面的基礎(chǔ)上修改
-
修改settings.py
ITEM_PIPELINES = { # 'Geely.pipelines.GeelyPipeline': 1, # 'scrapy.pipelines.images.ImagesPipeline': 1, 'Geely.pipelines.GeelyImagesPipeline': 1, } #工程根目錄 project_dir = os.path.dirname(__file__) #下載圖片存儲位置 IMAGES_STORE = os.path.join(project_dir, 'images') -
改寫pipelines,py
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html import os from urllib import request from scrapy.pipelines.images import ImagesPipeline from Geely import settings # class GeelyPipeline(object): # def __init__(self): # #os.path.dirname()獲取當(dāng)前文件的路徑,os.path.join()獲取當(dāng)前目錄并拼接成新目錄 # self.path = os.path.join(os.path.dirname(__file__), 'images') # # 判斷路徑是否存在 # if not os.path.exists(self.path): # os.mkdir(self.path) # def process_item(self, item, spider): # #分類存儲 # catagory = item['catagory'] # urls = item['image_urls'] # catagory_path = os.path.join(self.path, catagory) # #如果沒有該路徑即創(chuàng)建一個 # if not os.path.exists(catagory_path): # os.mkdir(catagory_path) # for url in urls: # #以_進(jìn)行切割并取最后一個單元 # image_name = url.split('_')[-1] # request.urlretrieve(url,os.path.join(catagory_path,image_name)) # return item # 繼承ImagesPipeline class GeelyImagesPipeline(ImagesPipeline): # 該方法在發(fā)送下載請求前調(diào)用锋爪，本身就是發(fā)送下載請求的 def get_media_requests(self, item, info): # super()直接調(diào)用父類對象 request_objects = super(GeelyImagesPipeline, self).get_media_requests(item, info) for request_object in request_objects: request_object.item = item return request_objects def file_path(self, request, response=None, info=None): path = super(GeelyImagesPipeline, self).file_path(request, response, info) # 該方法是在圖片將要被存儲時調(diào)用丙曙，用于獲取圖片存儲的路徑 catagory = request.item.get('catagory') # 拿到IMAGES_STORE images_stores = settings.IMAGES_STORE catagory_path = os.path.join(images_stores, catagory) #判斷文件名是否存在,如果不存在創(chuàng)建文件 if not os.path.exists(catagory_path): os.mkdir(catagory_path) image_name = path.replace('full/','') image_path = os.path.join(catagory+'/',image_name) return image_path -
讓項(xiàng)目跑起來：
$ scrapy crawl BoYue
將會得到與原來相同的結(jié)果！Ｆ浣尽？髁！
此文章同時同步到我的個人博客緣來來來 ? Python爬蟲 --- 2.5 Scrapy之汽車之家爬蟲實(shí)踐
