畢業(yè)離校到上海漂泊已經(jīng)有些時(shí)日了实苞，每天的生活還算正常，魔都標(biāo)配的房租烈疚，日常生活消費(fèi)黔牵，告誡來這里的每一位追求夢想的人兒不得不更加努力。加班的周末爷肝，寫了一個(gè)簡單的腳本猾浦，加了些注釋，主要想讓參考的同學(xué)了解些編寫代碼的規(guī)范灯抛。我們要像寫情書般優(yōu)雅的寫代碼金赦。周末愉快！对嚼！
# -*- coding: utf-8 -*-
#!/usr/bin/python3
import requests,re
import json
import pymongo
from requests.exceptions import RequestException
from multiprocessing import Pool
#from pathos.multiprocessing import ProcessingPool as Pool
'''
request modle usage:
1.set headers
2.verify response.status_code==200
3.GoodPlan: use RequestException
'''
def getOnePage(url):
headers={
'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
}
try:
response=requests.get(url,headers=headers)
if response.status_code==200:
return response.text
return None
except ResquestException:
return None
'''
regular matching item and set a generator
'''
def parseOnePage(html):
pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
+'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
+'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?<dd>',re.S)
items= re.findall(pattern,html)
for item in items:
print(item)
yield{
'index':item[0],
'image':item[1],
'title':item[2],
'actor':item[3].strip()[3:],
'time':item[4].strip()[5:],
'score':item[5]+item[6]
}
'''
write content to local file
'''
def writeToFile(content):
with open('result.txt','a',encoding='utf-8') as wf:
wf.write(json.dumps(content,ensure_ascii=False)+'\n')
wf.flush()
def main(offset):
'''
configure your Database
'''
myclient=pymongo.MongoClient('mongodb://192.168.88.114:27017/')
dblist=myclient.database_names()
mydb=myclient['home']
mycol=mydb['maoyan']
'''
set uri and to get html
'''
url='https://maoyan.com/board/4?offset='+str(offset)
html =getOnePage(url)
'''
insert data to database
'''
for item in parseOnePage(html):
try:
#writeToFile(item)
mycol.insert_one(item)
except:
print("check you datebase configure")
if __name__ == '__main__':
'''
create a thread pool
'''
pool =Pool()
pool.map(main,[i*10 for i in range(10)])
下面則是記錄一些Requests庫使用中的一些需要注意的地方：
1.遇到網(wǎng)站證書校驗(yàn)時(shí)夹抗，可以視情況使用如下方法解決問題。
import requests
from requests.packages import urllib3
urllib3.disable_warnings()
response =requests.get('https://www.12306.cn',verify=False)
print(response.status_code)
2.加載本地的證書文件
import requests
response = requests.get('https://www.12306.cn',cert=('/path/server.crt','/path/key'))
print(response.status_code)
3.設(shè)置代理
import requests
proxies={'http':"http://127.0.0.1:9743",'https':"https://127.0.0.1:9743"}
response = requests.get('https://taobao.com',proxies=proxies)
print(response.status_code)
4 設(shè)置的代理有密碼
import requests
proxies={'http':"http://user:password@127.0.0.1:9743",'https':"https://user:pw@127.0.0.1:9743"}
response = requests.get('https://taobao.com',proxies=proxies)
print(response.status_code)
5.requests 設(shè)置socks5代理
pip3 install "requests[socks]"
import requests
proxies={'http':"socks5://127.0.0.1:9742",'https':"socks5://127.0.0.1:9742"}
response = requests.get('https://taobao.com',proxies=proxies)
print(response.status_code)
6.超時(shí)設(shè)置,捕獲異常
import requests
from requests.exception import ReadTimeout
try:
response = requests.get('https://taobao.com',timeout=3)
print(response.status_code)
except ReadTimeout:
print("timeout")
7猪半。認(rèn)證設(shè)置
import requests
from requests.auth import HTTPBasicAuth
response = requests.get('https://taobao.com',auth=HTTPBasicAuth('user','123'))
print(response.status_code)
