我的結(jié)果
鏈接.png
詳情.png
我的代碼
from bs4 import BeautifulSoup
import requests, pymongo, re, time, urllib, socket, json, random
# 基本頁(yè)面
url = 'http://bj.58.com/shoujihao/{}/pn{}/'
# headers
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}
# 使用session
req_session = requests.session()
# 建立數(shù)據(jù)庫(kù)
client = pymongo.MongoClient('localhost', 27017)
walden = client['walden']
phone_test = walden['phone_test']
phone_info_test = walden['phone_info_test']
# 從列表頁(yè)面獲取手機(jī)號(hào)和鏈接秽褒，并存入數(shù)據(jù)庫(kù)，對(duì)應(yīng)第一個(gè)結(jié)果圖
def get_phone_url(url, page, who=1):
# 整理地址，注意who和page的位置
search_page = url.format(who, page)
web_data = req_session.get(search_page, headers = headers)
soup = BeautifulSoup(web_data.text, 'lxml')
# 如果頁(yè)碼超出范圍則pass
if soup.find('strong'):
# 考慮到selector結(jié)構(gòu)一樣炊邦，為了區(qū)分廣告條目编矾，這里采用了find_all搭配正則
tel_numbers = soup.find_all('li', logr=re.compile('.*-1$'))
tel_urls = soup.find_all('li', logr=re.compile('.*-1$'))
for tel_number, tel_url in \
zip(tel_numbers, tel_urls):
tel_number = tel_number.get_text().split()[0]
# 嵌套find_all
tel_url = tel_url.find_all('a', 't')[0].get('href')
# 寫入數(shù)據(jù)庫(kù)
data = {'Url': tel_url, 'Number': tel_number}
print(data)
phone_test.insert_one(data)
else:
pass
# 從詳情頁(yè)面獲取信息
def get_phone_info(url):
try:
web_data = req_session.get(url, headers=headers, timeout=6)
soup = BeautifulSoup(web_data.text, 'lxml')
# 以防404
four_o_four = '404' in soup.find('script', type='text/javascript').get('src').split('/')
if four_o_four:
pass
else:
title = ' '.join(soup.select('.col_sub h1')[0].get_text().split())
price = soup.select('.price')[0].get_text().strip()
time = soup.select('li.time')[0].get_text().strip()
# 不是所有頁(yè)面都有區(qū)域，這里用正則表達(dá)做了一個(gè)判斷
if re.findall('區(qū)域', str(soup.select('.col_sub'))):
region = soup.select('.su_con a')[0].get_text().strip()
else:
region = None
name = soup.select('a.tx')[0].get_text()
contact = soup.select('span.f20')[0].get_text().strip()
# 寫入數(shù)據(jù)庫(kù)
phone_info_test.insert_one({
'title': title,
'price': price,
'time': time,
'region': region,
'name': name,
'contact': contact
})
# 防止報(bào)錯(cuò)
except UnicodeDecodeError as e:
print('-----UnicodeDecodeError url:', url)
except urllib.error.URLError as e:
print("-----urlError url:", url)
except socket.timeout as e:
print("-----socket timout:", url)
# 從里【開始】馁害，獲取10頁(yè)鏈接寫入數(shù)據(jù)庫(kù)
for single_page in range(10):
print('頁(yè)數(shù)：',single_page + 1, '\n', '-'*60)
time.sleep(10)
get_phone_url(url, single_page)
# 從數(shù)據(jù)庫(kù)獲取鏈接
for item in phone_test.find():
print(item['Url'])
info_page = item['Url']
get_phone_info(info_page)
time.sleep(1)
我的感想：
- 我估計(jì)我花了5個(gè)多小時(shí)窄俏，處理了挺多大大小小的坑，主要的坑點(diǎn)在于：廣告的剔除碘菜，個(gè)別結(jié)構(gòu)差異的地方（頁(yè)面超出范圍凹蜈、404等）要做判斷，以及老是報(bào)錯(cuò)‘Max retries exceeded with url’
- 對(duì)于坑點(diǎn)：
-
我覺得我用正則來(lái)剔除廣告有點(diǎn)“霸王硬上弓”忍啸，別的同學(xué)通過(guò)篩選鏈接來(lái)處理挺簡(jiǎn)單明了仰坦，但需要一些細(xì)心和意識(shí)。
廣告.png
正常.png
從地址來(lái)剔除廣告.png
-
- 對(duì)于‘Max retries exceeded with url’呢计雌。以我目前的經(jīng)驗(yàn)：要么等會(huì)弄悄晃，要么調(diào)整sleep的時(shí)間，要么加proxy白粉，要么改headers传泊，屬于一種瞎子摸象的狀態(tài)。
- 對(duì)于結(jié)構(gòu)上的差異或許需要經(jīng)驗(yàn)來(lái)培養(yǎng)意識(shí)鸭巴，像404的排除眷细，頁(yè)碼范圍的超出，老師講出來(lái)順理成章鹃祖，但自己操作起來(lái)還是挺生疏的溪椎。
