import requests
from bs4 import BeautifulSoup
import re
jianshu_url='http://www.reibang.com'
base_url='http://www.reibang.com/u/54b5900965ea?order_by=shared_at&page=' #更改作者，可在此處修改?之前 /u之后的
user_agent='Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
headers={'User-Agent':user_agent}
pattern='http://www.reibang.com/users/6c7437065202/timeline' #此處/timeline 之前 /users之后的可以修改
articlePage=[]
f=open('run_write.txt','w',encoding='utf-8')
# articleList=[]
def download_page():
i = 1
while True:
r=requests.get(base_url+str(i),headers=headers)
articlePage.append(r.url)
if(r.url==pattern):break #判斷出是否已經(jīng)到達最后一個請求頁面雷客，如果是，則結(jié)束
print('這是第{}個請求的網(wǎng)頁'.format(i))
i += 1
get_article_from_page(r.url)
def get_article_from_page(url):
r=requests.get(url,headers=headers)
soup=BeautifulSoup(r.text,'lxml')
articleList=soup.find_all('li',id=re.compile(r'note-\d+'))
for i in range(len(articleList)):
articleTitle = articleList[i].find('a', class_='title').text
articleUrl=jianshu_url+articleList[i].find('a',class_='title')['href']
print(articleTitle+5*' '+articleUrl)
download_page()
2.png
