展示代碼###
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
import time
info = []
headers = {
'cookie': '_gat=1; gr_user_id=7e4ae08a-42a2-4645-bcdc-c5121ebf4c28; _knewone_v2_session=bUswaVJNb2poakxGZVUwRmQ0V0dqWm9sOFpQVDBjMWF1b0grazh0Q1l6dGpub3BHK1JSanV2OGc0ZmhDZitBMkNCZFY5T2JZQ1ZxN0ZpM0dONExsN3AwREtLcmw3dC9Ub1hKbnk5N3NabG1oTGdHU01EM0lHZU5nQWRGRGdaZUhhV1crYTY4Zlk4NGdUSjhmak1jSFVNSSt1YzNEOVY5bS9zVC8rbkJHT282M0dheXpwT0FNWDFoV1lZQ0hNcnp5LS1qbEM3TStTRDNVOFFLWkF3UEh6QzNBPT0%3D--62e609c4894070dfadfe1ddfd816eae455c0803e; _ga=GA1.2.1202148329.1464331142; Hm_lvt_b44696b80ba45a90a23982e53f8347d0=1464331143; Hm_lpvt_b44696b80ba45a90a23982e53f8347d0=1464331216; gr_session_id_e7b7e334c98d4530928513e7439f9ed2=65a4dbb6-0a13-4293-8745-06637ceba521',
'referer': 'https://knewone.com/things',
'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36',
'x-csrf-token': 'W+zjfj7CxkXZvvltEUXvvLWVrEMMdQZFgV7wgdQx5On71c+Iqo9YcmlK1u+Cdz7V92NR59vOyBQaOfzbMcKCeA==',
}
def getText(item):
return '' if item == 0 else ''.join(item.get_text().replace('\n', ''))
def getPic(item):
return '' if item == 0 else ''.join(item.get('src'))
def getUrl(item):
return '' if item == 0 else ''.join('https://knewone.com' + item.get('href'))
def getInfo(url, data=None):
web_data = requests.get(url, headers=headers)
web_data.encoding = "utf-8"
if web_data.status_code == 200:
soup = BeautifulSoup(web_data.text, 'html.parser')
titles = soup.select('#things_list > article > section > h4 > a')
pics = soup.select('#things_list > article > header > a > img')
favos = soup.select('span.fanciers_count')
links = soup.select('#things_list > article > header > a')
if data == None:
for title, pic, favo,link in zip(titles, pics, favos,links):
time.sleep(2)
data = {
'title': getText(title),
'pic': getPic(pic),
'favo': getText(favo),
'link': getUrl(link)
}
print(data)
def get_more_page(start, end):
url = 'https://knewone.com/things?page='
for i in range(start, end):
getInfo(url + str(i))
get_more_page(1, 50)
出現(xiàn)問題###
gbk編碼.png
這是一個(gè)老生常談的問題梆奈，最主要是因?yàn)閜ycharm的編碼有問題莫鸭，只需要將IDE和Project 的編碼問題都改成UTF-8即可汰聋。
UTF-8編碼.png
這個(gè)問題是個(gè)大坑(找了好久我才搞定的/(ㄒoㄒ)/~~)精偿。只要網(wǎng)頁中有特殊字符棘劣，如 ° ，[B?hm]等离赫，抓取的時(shí)候就都會報(bào)錯(cuò)芭逝。
另外的附加解決方案是
a.加入shebang,他也只是解決IDE當(dāng)中的編碼問題
#!/usr/bin/env python
# -*- coding: utf-8 -*-
b.給網(wǎng)頁進(jìn)行編碼轉(zhuǎn)換，適用于輸出時(shí)有亂碼渊胸。尤其需要注意網(wǎng)頁的編碼形式旬盯，盡管大多數(shù)為UTF-8
web_data = requests.get(url, headers=headers)
web_data.encoding = "utf-8"
2.還有一個(gè)小坑，對tag使用.get() or .get_test()翎猛，有些需要加[0]胖翰，但是有些不需要
