如何下載圖片
# 圖片的下載
import requests
url = '[http://g.hiphotos.baidu.com/image/pic/item/5366d0160924ab18014cefd83bfae6cd7a890b82.jpg](qq://txfile/#)'
response = requests.get(url)
# 獲取bytes類型的響應(yīng)
data = response.content
with open('風(fēng)景.jpg','wb') as f:
f.write(data)
如何翻頁
觀察網(wǎng)頁翻頁后url的變化执隧，運(yùn)用range和參數(shù)傳遞實(shí)現(xiàn)翻頁蜈七。
for i in range(0, 250, 25):
url = 'https://movie.douban.com/top250?start={}&filter='.format(i)
25為每一頁展示的電影數(shù)量；start=x表示當(dāng)前頁面從序號x的電影開始穷娱。
1. 豆瓣電影TOP250爬蟲
-
html源碼
豆瓣TOP250源碼 代碼實(shí)現(xiàn)
import requests
from lxml import etree
def spider_douban_top250():
movie_list_info = []
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
#觀察網(wǎng)頁翻頁url的變化讯屈，運(yùn)用range和參數(shù)傳遞實(shí)現(xiàn)翻頁
for i in range(0, 250, 25):
url = 'https://movie.douban.com/top250?start={}&filter='.format(i)
# 獲取bytes類型響應(yīng)
data = requests.get(url, headers=headers).content
html = etree.HTML(data)
ol_list = html.xpath('//div[@id="content"]//div[@class="article"]/ol/li')
for movie in ol_list:
# 影片序號
serial_number = movie.xpath('./div[@class="item"]/div[@class="pic"]/em/text()')
if len(serial_number) == 0:
serial_number = ''
else:
serial_number = serial_number[0]
# print(serial_number)
# 電影的名字
movie_name = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="hd"]/a/span[1]/text()')[0]
# print(movie_name)
# 電影的介紹
movie_introduce = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/p[1]/text()')[
0].strip()
# print(movie_introduce)
# 電影的星級
star = \
movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[2]/text()')[0]
# print(star)
# 電影的評價(jià)
evalute = movie.xpath(
'./div[@class="item"]/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[4]/text()')
evalute = evalute[0].replace('人評價(jià)', '')
# print(evalute)
# 電影的描述
describe = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/p[@class="quote"]/span[1]/text()')
# print(describe)
# 電影封面的地址
movie_img_url = movie.xpath('./div[@class="item"]/div[@class="pic"]/a/img/@src')[0]
# print(movie_img_url)
movie_list_info.append({
'serial_number': serial_number,
'movie_name': movie_name,
'movie_introduce': movie_introduce,
'star': star,
'evalute': evalute,
'describe': describe,
'movie_img_url': movie_img_url
})
for movie in movie_list_info:
print(movie)
# 下載圖片
for movie in movie_list_info:
url = movie['movie_img_url']
resp = requests.get(url)
if resp.status_code == 200:
img_name = '0000000{}.jpg'.format(movie['serial_number'])
with open('./imgs/{}'.format(img_name), 'wb') as f:
f.write(resp.content)
spider_douban_top250()
這里使用的是lxml.etree而不是之前的lxml.html蛋哭，得到結(jié)果基本一致，但etree功能更多涮母。
-
爬取結(jié)果
爬取的電影信息
下載的圖片
2. 爬取7k7k游戲信息
-
7k7k網(wǎng)頁源碼
7k7k源碼 - 代碼實(shí)現(xiàn)
import requests
from lxml import etree
def spider_7k7k_top100():
game_list_info = []
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
#爬取10頁
for i in range(0, 10):
url = 'http://www.7k7k.com/tag/134/index_{}.htm#p-anchor'.format(i)
# 獲取bytes類型響應(yīng)
data = requests.get(url, headers=headers).content
html = etree.HTML(data)
ul_list = html.xpath('//div[@id="recommend-new-most"]//div[@class="ui-slide-cont"]/ul/li')
#print(ul_list)
for game in ul_list:
# 游戲名字
game_name = game.xpath('./a[@class="li-top-a"]/span/text()')
print(game_name)
# 游戲封面的地址
imgs = game.xpath('./a[@class="li-top-a"]/img/@src')
if imgs:
game_img_url = game.xpath('./a[@class="li-top-a"]/img/@src')[0]
print(game_img_url)
# 游戲鏈接
game_link = 'http://www.7k7k.com{}'.format(game.xpath('./a[@class="li-top-a"]/@href')[0])
#game_link = game.xpath('./a[@class="li-top-a"]/@href')[0]
print(game_link)
game_list_info.append({
'game_name': game_name,
'game_link': game_link,
'game_img_url': game_img_url
})
for game in game_list_info:
print(game)
#下載圖片
index = 1; #圖片索引
for game in game_list_info:
url = game['game_img_url']
resp = requests.get(url)
if resp.status_code == 200:
img_name = '0000000{}.jpg'.format(i)
with open('./imgs/{}'.format(img_name), 'wb') as f:
f.write(resp.content)
i += 1
spider_7k7k_top100()
-
爬取結(jié)果
爬取結(jié)果 -
游戲封面
下載游戲封面
