爬取knewone上的信息
<pre>
author = 'LEE'
-- coding: utf-8 -
from bs4 import BeautifulSoup
import time
import requests
import time
import io
import sys
import urllib.request
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')
url = 'https://knewone.com/things/?page='
def get_page(url,data=None):
wb_data = requests.get(url)
soup = BeautifulSoup(wb_data.text,'lxml')
imgs = soup.select('a.cover-inner > img')
titles =soup.select('section.content > h4 > a')
links = soup.select('section.content > h4 > a')
#print(soup)
if data ==None:
for img,title,link in zip(imgs,titles,links):
data = {
'img':img.get('src'),
'title':title.get('title'),
'link':link.get('href'),
}
print(data)
def get_more_page(start,end):
for one in range(start,end):
get_page(url+str(one))
time.sleep(1)
get_more_page(1,10)
</pre>
