第一周實戰(zhàn)作業(yè)
學習python的第一周 5.19號完成練習實戰(zhàn)作業(yè) 爬去58同城平板電腦列表里的賣家信息
Paste_Image.png
目標：爬去賣家以及商品的基本信息
代碼如下：
import requests
from bs4 import BeautifulSoup
import time
def get_message(url):
#print(url)
wb_date = requests.get(url)
id = (url.split('/')[-1])[:14]#截取商品ID得到對應的id
views = get_totols(id)
soup = BeautifulSoup(wb_date.text,'lxml')
categories = soup.select('#header > div.breadCrumb.f12 > span:nth-of-type(3) > a')#類目
titles = soup.select('#content > div.person_add_top.no_ident_top > div.per_ad_left > div.col_sub.mainTitle > h1')#標題
times = soup.select('#index_show > ul.mtit_con_left.fl > li.time')#發(fā)布時間
prices = soup.select('#content > div.person_add_top.no_ident_top > div.per_ad_left > div.col_sub.sumary > ul > li:nth-of-type(1) > div.su_con > span')#價格
olds = soup.select('#content > div.person_add_top.no_ident_top > div.per_ad_left > div.col_sub.sumary > ul > li:nth-of-type(2) > div.su_con > span')#成色
areas1 = soup.select('#content > div.person_add_top.no_ident_top > div.per_ad_left > div.col_sub.sumary > ul > li:nth-of-type(3) > div.su_con > span > a:nth-of-type(1)')#區(qū)域1
areas2 = soup.select('#content > div.person_add_top.no_ident_top > div.per_ad_left > div.col_sub.sumary > ul > li:nth-of-type(3) > div.su_con > span > a:nth-of-type(2)')#區(qū)域2
for category,title,time,price,old,area1,area2 in zip(categories,titles,times,prices,olds,areas1,areas2):
data = {
'類目':category.get_text(),
'標題':title.get_text(),
'發(fā)布時間':time.get_text(),
'價格':price.get_text(),
'成色':old.get_text().strip(),
'區(qū)域':area1.get_text()+'-'+area2.get_text(),
'瀏覽量':views
}
print(data)
return None
#去除列表中重復連接
def delRepeat(list):
for x in list:
while list.count(x)>1:
del list[list.index(x)]
#print(list)
return list
#獲取有效連接列表
def get_links(url):
links = []
wb_data = requests.get(url)
soup = BeautifulSoup(wb_data.text,'lxml')
href = soup.select(' tr > td.t > a')
for link in href:
if 'http://bj.58.com/pingbandiannao/' in link.get('href'):#判斷路徑里是否有關鍵字 去除推薦的和轉轉的連接
links.append(link.get('href'))#將連接加入到列表中去
links = delRepeat(links)#去重
for wb_link in links:
get_message(wb_link)
time.sleep(2) # 防止反爬
#獲取瀏覽量的方法
def get_totols(id):
headers ={
'Referer':'http://bj.58.com/pingbandiannao/25390255065933x.shtml?adtype=1&PGTID=0d305a36-0000-1576-ef5f-2547f6476ccf&entinfo=25390255065933_0&psid=132996494191831950071360497&iuType=q_2&ClickID=2',#58瀏覽量針對頭部referer有判斷 不偽造信息拿到值為0
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36',
'Cookie':'f=n; userid360_xml=BE5F745CB21889493F55E2AB183C6316; time_create=1466153186464; bj58_id58s="ZGFzSVk5dkFyUDZQODU4OA=="; id58=c5/njVc0K1eW7SWOA1IbAg==; sessionid=895a33d3-6b04-4820-b906-89323e0a3419; als=0; myfeet_tooltip=end; bdshare_firstime=1463552754347; ipcity=sh%7C%u4E0A%u6D77; 58home=sh; f=n; bj58_new_session=0; bj58_init_refer=""; bj58_new_uv=3; 58tj_uuid=0f55bf86-b266-4866-aea3-3fed2ffe6ccf; new_session=0; new_uv=2; utm_source=; spm=; init_refer=http%253A%252F%252Fbj.58.com%252Fpbdn%252F0%252F; final_history=26044978248654%2C26044268155977%2C26013739776688'
}
api = 'http://jst1.58.com/counter?infoid={}&userid=&uname=&sid=516903216&lid=1&px=&cfpath=5,38484'.format(id)
js = requests.get(api,headers=headers)
views = js.text.split('=')[-1]
#print(views)
return views
full_url = ['http://bj.58.com/pbdn/{}/'.format(str(i)) for
i in range(0, 10, 1)]
for link in full_url:
get_links(link)
爬取結果：
Paste_Image.png
難點：本次項目是第一次比較全面的爬取項目髓废，難點由以下幾點
-.商品信息瀏覽量這個字段 58是用js來控制
-.抓取到這個JS的URL后發(fā)現(xiàn)58對這個URL做過反爬取 必須偽造頭部信息中的Referer才能得到返回totol這個瀏覽量
總結：通過一周學習收獲：由于編寫本次抓取前构捡，不知曉后面有講解視頻，自己一個人摸索，之后完成后再看了老師視頻后發(fā)現(xiàn)有許多地方都能優(yōu)化處理恳谎，但是這次的爬取也給自己增加了不少的信心和樂趣敦锌！
