今天開始爬取今日頭條的第二個入口搜索佣蓉，搜索有四個tab分別為綜合，視頻亲雪，圖集勇凭，用戶。先來分析一下綜合的搜索接口
https://www.toutiao.com/search_content/?offset=0&format=json&keyword=%E7%A4%BE%E5%8C%BA%E6%96%B0%E9%9B%B6%E5%94%AE&autoload=true&count=20&cur_tab=1&from=search_tab
只需要傳遞keyword 搜索關(guān)鍵字即可匆光，無需進行其他處理套像，直接解析接口返回數(shù)據(jù)，該接口返回的數(shù)據(jù)有點雜數(shù)據(jù)結(jié)構(gòu)不統(tǒng)一终息，有文章夺巩，用戶，微頭條周崭，問答柳譬，搜索推薦，其他關(guān)鍵字搜索推薦续镇，在做數(shù)據(jù)解析的時候要針對這集中數(shù)據(jù)結(jié)構(gòu)分別做處理美澳。老樣子直接上代碼：
def get_search_article(self, keyword, offset=0):
keyword = urllib.request.quote(keyword)
req_url = "https://www.toutiao.com/search_content/?offset={}&format=json&keyword={}&autoload=true&count=20&cur_tab=1&from=search_tab".format(offset,keyword)
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
'Connection': 'keep-alive',
'authority': 'www.toutiao.com',
'referer': "https://www.toutiao.com/search/?keyword={}".format(keyword),
'method': 'GET',
'path': "/search_content/?offset={}&format=json&keyword={}&autoload=true&count=20&cur_tab=1&from=search_tab".format(offset,keyword),
'scheme': 'https'
}
self.s.headers.update(headers)
req = self.s.get(req_url, proxies=get_proxy_ip())
time.sleep(random.random() * 2 + 3)
data = json.loads(req.text)
items = data['data']
if data['has_more'] == 1:
self.page = self.page + 1
offset = 20 * self.page
self.parse_data(items)
time.sleep(2)
self.get_search_article(keyword, offset)
else:
self.parse_data(items)
toutiaodb.save(self.search_item_list)
def parse_data(self, items):
for item in items:
try:
type = item['cell_type']
except:
type = 0
if type == 37: #微頭條
pass
elif type == 50:
pass
elif type == 66:
pass
elif type == 26: #內(nèi)容推薦
pass
elif type == 20: #搜索推薦
pass
elif type == 38: #用戶
pass
else:
titem = toutiaoitem()
titem.user_id = item['user_id']
try:
titem.source = item['source']
except:
titem.source = item['name']
titem.title = item['title']
titem.source_url = item['article_url']
titem.media_url = item['media_url']
titem.item_id = item['item_id']
titem.abstract = item['abstract']
titem.comments_count = item['comments_count']
titem.behot_time = item['behot_time']
titem.image_url = item['image_url']
titem.image_list = item['image_list']
titem.tag = item['tag']
if 'play_effective_count' in item:
titem.article_genre = 'vedio'
titem.read_count = item['play_effective_count']
else:
titem.article_genre = 'article'
self.search_item_list.append(titem)
