直接附上代碼:
import urllib.request
import urllib.parse
import jsonpath
import json
import re
import time
'''
ajax接口
https://rate.taobao.com/feedRateList.htm?auctionNumId=593014213362&userNumId=903511695¤tPageNum=2&pageSize=20
'''
items = []
def handle_request(url, page):
url = url.format(page)
# print(url)
# 必須加COOKIE,不然json返回不回來
headers = {
'Accept': '*/*',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Referer': 'https://item.taobao.com/item.htm?spm=a230r.1.14.31.1a9df9eaZ2TtQ2&id=593014213362&ns=1&abbucket=4',
'Cookie': 'miid=1034753504511266755;cna=dx1nFLT5mQcCAXWaR2QxwmBW;'
'thw=cn;t=325ff4e457909046382b22a33c83eadb;tracknick=acc%5Cu80E1%5Cu4F20;'
'lgc=acc%5Cu80E1%5Cu4F20;tg=0;v=0;'
'cookie2=128cea00303f12de470490c850cc00b1;_tb_token_=fdb4b335af83e;'
'unb=2701405241;sg=%E4%BC%A01a;_l_g_=Ug%3D%3D;skt=051d1e62c9ab8dc5;'
'cookie1=BxMH3NLGjomw6rJWdejLCTcGCFEkYwy%2BoQnBejFVBZI%3D;csg=93fdcb74;'
'uc3=vt3=F8dBy3jYcBxWmWeUeDY%3D;id2=UU8IPT1oZbApWA%3D%3D;nk2=AnIsGxSURQ%3D%3D;lg2=Vq8l%2BKCLz3%2F65A%3D%3D;'
'existShop=MTU1OTY1MDAxNw%3D%3D;_cc_=U%2BGCWk%2F7og%3D%3D;dnk=acc%5Cu80E1%5Cu4F20;_nk_=acc%5Cu80E1%5Cu4F20;'
'cookie17=UU8IPT1oZbApWA%3D%3D;mt=ci=6_1&np=;'
'enc=UFEUuOrXKTfwHDdKWP%2F9JacRMgPfTTQxKHxhkY8FExWK0%2FOKPqyryg048bj3QRjP%2Blwvl2%2BmU0N0pIBcrlD2PA%3D%3D;'
'hng=CN%7Czh-CN%7CCNY%7C156;'
'uc1=cookie15=VT5L2FSpMGV7TQ%3D%3D&cookie14=UoTZ7Yy7WSTGKg%3D%3D;'
'_m_h5_tk=3627342e3b09ae88a2c724b51c503d85_1559657695270;'
'_m_h5_tk_enc=c88d8e3f4ba68bc083f853e956656b6c;'
'x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0;'
'x5sec=7b22726174656d616e616765723b32223a22623034623263306164386231666234336634303330663463646430663064653743504c4f32656346454e364e6b736a586d664c7373774561444449334d4445304d4455794e4445374d513d3d227d;'
'whl=-1%260%260%261559652412550;'
'l=bB_eivGRvlAmYuqwBOfgiuI8a17TwIdfhsPzw4_G2ICP_bfDwBcPWZTPLZTkC3GVa6GXR3oyuHLBBW8iLyUCh;'
'isg=BGNjWHeLO7xJIPCeqwvJFNPV8qfN8Pf9ajjwFJXBBkM21IT2HSu26rnCyuT_9E-S',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
'Connection': 'keep-alive',
'Host': 'rate.taobao.com',
}
request = urllib.request.Request(url=url, headers=headers)
return request
def json_parse(json_content):
# 將json格式兩邊的小括號干掉,使用正則替換方式
json_content = re.sub(r'\(', '', json_content)
json_content = re.sub(r'\)', '', json_content)
# 方式二壳咕,使用字符串方式去掉小括號,將json格式兩邊的小括號和非字符干掉
# json_content=json_content.strip('() \n\t\r')
# print(json_content)
# 將json格式字符串轉(zhuǎn)化為python對象
py_json = json.loads(json_content)
comments_list = py_json['comments']
for comment in comments_list:
# 使用jsonpath方式查找
user = jsonpath.jsonpath(comment, '$..user')[0]
# 用戶頭像
user_face = 'http;' + user['avatar']
# print(user_face)
#用戶名
user_name=user['nick']
# print(user_name)
#評論內(nèi)容
user_text=comment['content']
#評論日期
user_date=comment['date']
#手機(jī)信息
info=jsonpath.jsonpath(comment, '$..sku')[0]
# up_date=comment['date']
# up_content=comment['content']
# up_modol=comment['auction']['sku']
# up_author=comment['user']['nick']
item={
'評論人': user_name,
'評論圖像':user_face,
'評論時間': user_date,
'評論內(nèi)容': user_text,
'購買手機(jī)類型': info,
}
items.append(item)
def main():
url = 'https://rate.taobao.com/feedRateList.htm?auctionNumId=593014213362&userNumId=903511695¤tPageNum={}&pageSize=20'
start_page = int(input('請輸入爬取開始頁面:'))
end_page = int(input('請輸入爬取結(jié)束頁面:'))
for page in range(start_page, end_page + 1):
request = handle_request(url, page)
json_content = urllib.request.urlopen(request).read().decode()
# print(json_content)
time.sleep(2)
json_parse(json_content)
string = json.dumps(items, ensure_ascii=False)
# 以TXT格式保存到本地
with open('taobao.txt', 'w', encoding='utf8') as fp:
fp.write(string)
if __name__ == '__main__':
main()