做爬蟲的同學(xué)應(yīng)該經(jīng)常遇到登錄驗(yàn)證的問題，在這里我使用phantomejs模擬登錄并且保存cookies损晤，方便再次登錄脆粥。廢話不多說砌溺，直接切入正題。
-
模擬登錄
def login(username, password, driver=None):
url = 'https://www.jd.com/productDetail.html'
driver.get(url)
driver.find_element_by_xpath('//input[@id="loginname"]').send_keys(username)
logger.info ('已輸入賬號(hào)')
driver.find_element_by_xpath('//input[@id="nloginpwd"]').send_keys(password)
logger.info ('已輸入密碼')
driver.find_element_by_xpath('//a[@id="loginsubmit"]').click()
logger.info ('已點(diǎn)擊登錄按鈕')
return driver
-
從網(wǎng)上獲取cookies
def get_cookies_from_network():
driver = login(username, password, driver)
# 獲取cookies保存到本地并且返回此cookies
cookies = driver.get_cookies()
cookie_dict = {}
logger.info(cookies)
for cookie in cookies:
if 'name' in cookie.keys() and 'value' in cookie.keys():
cookie_dict[cookie['name']] = cookie['value']
logger.info(cookie_dict)
with open('cookies.txt', 'w') as f:
# 保存cookies到本地
f.write(json.dumps(cookies))
logger.info("保存cookis成功.")
return cookie_dict
-
從本地緩存獲取cookies
def get_cookies_from_cache():
cookies_dict = {}
if os.path.exists('cookies.txt'):
# 如果本地有cookies文件变隔，則讀取本地cookies规伐，否則返回空
with open('cookies.txt', 'r') as f:
for i in json.loads(f.read()):
if 'name' in i.keys() and 'value' in i.keys():
if 'expiry' in i.keys():
expiry_date = int(i['expiry'])
# 判斷cookies是否過期，過期則返回空
if expiry_date > (int)(time.time()):
cookies_dict[i['name']] = i['value']
else:
logger.info('過期啦：{}'.format(i['name']))
return {}
else:
cookies_dict[i['name']] = i['value']
else:
return cookies_dict
logger.info(cookies_dict)
return cookies_dict
-
獲取cookies時(shí)匣缘，優(yōu)先訪問本地的cookies猖闪，如果cookies過期則再次獲取。
def get_cookies():
# 先從本地獲取cookies
cookie_dict = get_cookies_from_cache()
if not cookie_dict:
# 從本地返回的cookies為空則從網(wǎng)上獲取cookies
cookie_dict = self.get_cookies_from_network()
return cookie_dict
-
帶著cookies登錄網(wǎng)站
def get_content(driver=None):
#獲取cookies
cookies = get_cookies()
# 先訪問一遍目標(biāo)網(wǎng)站
driver.get(url)
# 刪除當(dāng)前的cookies
driver.delete_all_cookies()
for k,v in cookies.items():
# 添加cookies
driver.add_cookie({'name': k, 'value': v})
time.sleep(1)
logger.info('get url again')
# 再次訪問目標(biāo)網(wǎng)站肌厨，模擬登錄成功
driver.get(self.url)
logger.info(driver.page_source)
