王者榮耀英雄信息爬取

分析

入口頁(yè)面地址

https://pvp.qq.com/web201605/herolist.shtml

第一步獲取所有英雄的列表

image-20200525100704049.png

可以看到英雄列表是在源碼中可以被找到的

image-20200525100755019.png

第二步 獲取英雄的各種信息

英雄的基本信息放在一個(gè)class = "cover"的div中 我們主要采集英雄的名稱 和 技能介紹

image-20200525101942363.png

技能部分都在 class=" zk-con3 zk-con" 中 中的 ul中

image-20200525103312535.png

爬取英雄列表

創(chuàng)建工程

scrapy startproject wzry
cd wzry

創(chuàng)建爬蟲(chóng)

scrapy genspider wzry_spider pvp.qq.com

修改配置

# 不遵循robots協(xié)議 因?yàn)檎军c(diǎn)沒(méi)有這個(gè)文件 爬蟲(chóng)會(huì)直接略過(guò)
ROBOTSTXT_OBEY = False
# 添加請(qǐng)求頭
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
}

# 下載延遲
DOWNLOAD_DELAY = 1

修改初始頁(yè)面

start_urls = ['https://pvp.qq.com/web201605/herolist.shtml']

創(chuàng)建爬取列表方法(測(cè)試)

    def parse(self, response):
        print("=" * 50)
        print(response)
        print("=" * 50)

運(yùn)行爬蟲(chóng)

scrapy crawl wzry_spider

2020-05-25 11:08:53 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
2020-05-25 11:08:54 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://pvp.qq.com/web201605/herolist.shtml> (referer: None)
==================================================
<200 https://pvp.qq.com/web201605/herolist.shtml>
==================================================
2020-05-25 11:08:54 [scrapy.core.engine] INFO: Closing spider (finished)
2020-05-25 11:08:54 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 315,

成功返回結(jié)果

爬取所有英雄鏈接方法

class WzrySpiderSpider(scrapy.Spider):
    name = 'wzry_spider'
    allowed_domains = ['pvp.qq.com']
    start_urls = ['https://pvp.qq.com/web201605/herolist.shtml']    # 起始url
    base_url = "https://pvp.qq.com/web201605/"  # url 前綴

    def parse(self, response):
        print("=" * 50)
        # print(response.body)
        hero_list = response.xpath("http://ul[@class='herolist clearfix']//li")
        for hero in hero_list:
            url = self.base_url + hero.xpath("./a/@href").get()
            print(url)
            # yield scrapy.Request(url)
        print("=" * 50)

https://pvp.qq.com/web201605/herodetail/114.shtml
https://pvp.qq.com/web201605/herodetail/113.shtml
https://pvp.qq.com/web201605/herodetail/112.shtml
https://pvp.qq.com/web201605/herodetail/111.shtml
https://pvp.qq.com/web201605/herodetail/110.shtml
https://pvp.qq.com/web201605/herodetail/109.shtml
https://pvp.qq.com/web201605/herodetail/108.shtml
https://pvp.qq.com/web201605/herodetail/107.shtml
https://pvp.qq.com/web201605/herodetail/106.shtml
https://pvp.qq.com/web201605/herodetail/105.shtml
==================================================

爬取英雄詳情頁(yè)面

• 獲得基本信息

# 基本信息塊
hero_info = response.xpath("http://div[@class='cover']")
hero_name = hero_info.xpath(".//h2[@class='cover-name']/text()").get()
print(hero_name)
# 分類
sort_num = hero_info.xpath(".//span[@class='herodetail-sort']/i/@class").get()[-1:]
print(sort_num)
# 生存能力
viability = hero_info.xpath(".//ul/li[1]/span/i/@style").get()[6:]
print("生存能力:" + viability)
# 傷害
aggressivity = hero_info.xpath(".//ul/li[2]/span/i/@style").get()[6:]
print("攻擊能力:" + aggressivity)
effect = hero_info.xpath(".//ul/li[3]/span/i/@style").get()[6:]
print("技能影響" + effect)
difficulty = hero_info.xpath(".//ul/li[4]/span/i/@style").get()[6:]
print("上手難度:" + difficulty)

• 獲取技能信息

 skill_list = response.xpath("http://div[@class='skill-show']/div[@class='show-list']")

        for skill in skill_list:
            skill_name = skill.xpath("./p[@class='skill-name']/b/text()").get()
            if not skill_name:
                continue
            # 冷卻時(shí)間
            cooling = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("：")[1].strip().split('/')
            # 消耗
            consume = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("：")[1].strip().split('/')
            # "".strip()
            # 如果這個(gè)技能是空的就 continue

            # 技能介紹
            skill_desc = skill.xpath("./p[@class='skill-desc']/text()").get()
            new_skill = {
                "name": skill_name,
                "cooling": cooling,
                "consume": consume,
                "desc": skill_desc
            }

                new_hero = HeroInfo(name=hero_name,
                            sort_num=sort_num,
                            viability=viability,
                            aggressivity=aggressivity,
                            effect=effect,
                            difficulty=difficulty,
                            skills_list=new_skill)

                yield new_hero

        

• 英雄信息item類

class HeroInfo(scrapy.Item):
    # 存字符串
    name = scrapy.Field()
    sort_num = scrapy.Field()
    viability = scrapy.Field()
    aggressivity = scrapy.Field()
    effect = scrapy.Field()
    difficulty = scrapy.Field()
    # 存字典
    skills_list = scrapy.Field()

• 儲(chǔ)存 pipelines.py文件

需要修改 ITEM_PIPELINES 配置 加入這個(gè)處理類

# -*- coding: utf-8 -*-

from scrapy.exporters import JsonItemExporter


class WzryPipeline:
    def __init__(self):
        # 打開(kāi)文件并實(shí)例化JsonItemExporter
        self.fp = open('result.json', 'wb')
        self.save_json = JsonItemExporter(self.fp, encoding="utf-8", ensure_ascii=False, indent=4)
        # 開(kāi)始寫入
        self.save_json.start_exporting()

    def open_spider(self, spider):
        pass

    def close_spider(self, spider):
        # 結(jié)束寫入
        self.save_json.finish_exporting()
        # 關(guān)閉文件
        self.fp.close()

    def process_item(self, item, spider):
        # 寫入item
        self.save_json.export_item(item)
        return item

運(yùn)行爬蟲(chóng)

scrapy crawl wzry_spider

結(jié)果

image-20200525150137203.png

全部代碼

wzry_spider.py

# -*- coding: utf-8 -*-
import scrapy
from wzry.items import HeroInfo


class WzrySpiderSpider(scrapy.Spider):
    name = 'wzry_spider'
    allowed_domains = ['pvp.qq.com']
    start_urls = ['https://pvp.qq.com/web201605/herolist.shtml']  # 起始url
    base_url = "https://pvp.qq.com/web201605/"  # url 前綴

    def parse(self, response):
        # print(response.body)
        hero_list = response.xpath("http://ul[@class='herolist clearfix']//li")
        for hero in hero_list:
            url = self.base_url + hero.xpath("./a/@href").get()
            yield scrapy.Request(url, callback=self.get_hero_info)

    def get_hero_info(self, response):
        # 基本信息塊
        global new_skill
        hero_info = response.xpath("http://div[@class='cover']")
        hero_name = hero_info.xpath(".//h2[@class='cover-name']/text()").get()

        # 分類
        sort_num = hero_info.xpath(".//span[@class='herodetail-sort']/i/@class").get()[-1:]

        # 生存能力
        viability = hero_info.xpath(".//ul/li[1]/span/i/@style").get()[6:]
        # 傷害
        aggressivity = hero_info.xpath(".//ul/li[2]/span/i/@style").get()[6:]
        effect = hero_info.xpath(".//ul/li[3]/span/i/@style").get()[6:]
        difficulty = hero_info.xpath(".//ul/li[4]/span/i/@style").get()[6:]
        # 技能列表
        skill_list = response.xpath("http://div[@class='skill-show']/div[@class='show-list']")

        for skill in skill_list:
            skill_name = skill.xpath("./p[@class='skill-name']/b/text()").get()
            if not skill_name:
                continue
            # 冷卻時(shí)間
            cooling = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("："
                                                                                         "")[1].strip().split('/')
            # 消耗
            consume = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("："
                                                                                         "")[1].strip().split('/')
            # "".strip()
            # 如果這個(gè)技能是空的就 continue

            # 技能介紹
            skill_desc = skill.xpath("./p[@class='skill-desc']/text()").get()
            new_skill = {
                "name": skill_name,
                "cooling": cooling,
                "consume": consume,
                "desc": skill_desc
            }

        new_hero = HeroInfo(name=hero_name,
                            sort_num=sort_num,
                            viability=viability,
                            aggressivity=aggressivity,
                            effect=effect,
                            difficulty=difficulty,
                            skills_list=new_skill)

        yield new_hero

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class WzryItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


class HeroInfo(scrapy.Item):
    # 存字符串
    name = scrapy.Field()
    sort_num = scrapy.Field()
    viability = scrapy.Field()
    aggressivity = scrapy.Field()
    effect = scrapy.Field()
    difficulty = scrapy.Field()
    # 存字典
    skills_list = scrapy.Field()

pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy.exporters import JsonItemExporter


class WzryPipeline:
    def __init__(self):
        self.fp = open('result.json', 'wb')
        self.save_json = JsonItemExporter(self.fp, encoding="utf-8", ensure_ascii=False, indent=4)
        self.save_json.start_exporting()

    def open_spider(self, spider):
        pass

    def close_spider(self, spider):
        self.save_json.finish_exporting()
        self.fp.close()

    def process_item(self, item, spider):
        self.save_json.export_item(item)
        return item