寫在最前面：
本文只做學(xué)習(xí)交流用筝野，請(qǐng)勿惡意使用相關(guān)代碼
測(cè)試代碼，注意限制請(qǐng)求次數(shù)
測(cè)試代碼粤剧，注意限制請(qǐng)求次數(shù)
測(cè)試代碼歇竟，注意限制請(qǐng)求次數(shù)
重要的事情說三遍。抵恋。焕议。
一、判斷一個(gè)網(wǎng)站是否采用Ajax動(dòng)態(tài)加載數(shù)據(jù)
- 網(wǎng)頁(yè)未刷新（URL未改變）弧关，網(wǎng)頁(yè)內(nèi)容發(fā)生變化
- Ajax的請(qǐng)求一般都會(huì)帶上X-Requested-With頭域盅安。
- 在請(qǐng)求列表中找到包含“Ajax.json”字符的請(qǐng)求
Ajax動(dòng)態(tài)加載頁(yè)面.png
二、爬取Ajax動(dòng)態(tài)加載數(shù)據(jù)的兩種方式：
- 直接分析ajax調(diào)用的接口,然后通過代碼請(qǐng)求這個(gè)接口
- 使用selenium+driver（瀏覽器驅(qū)動(dòng)）模擬瀏覽器行為獲取數(shù)據(jù)
三世囊、爬取拉勾網(wǎng)Python相關(guān)職位信息示例代碼：
1.導(dǎo)入相關(guān)庫(kù)
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests
from lxml import etree
import time
import re
import csv
import random
2.解析頁(yè)面别瞭，獲取職位數(shù)據(jù)
def parse_detail_position(page_html):
'''解析詳情頁(yè)面，返回一個(gè)職位的信息字典'''
html = etree.HTML(page_html)
department = html.xpath('//div[@class="position-head"]//div[@class="company"]/text()')[0].strip()
job_name = html.xpath('//div[@class="position-head"]//span[@class="name"]/text()')[0].strip()
salary = html.xpath('//dd[@class="job_request"]/p/span[1]/text()')[0].strip()
city = html.xpath('//dd[@class="job_request"]/p/span[2]/text()')[0].strip()
city = re.sub(r'[/ ]', '', city)
work_year = html.xpath('//dd[@class="job_request"]/p/span[3]/text()')[0].strip()
work_year = re.sub(r'[/ ]', '', work_year)
educ = html.xpath('//dd[@class="job_request"]/p/span[4]/text()')[0].strip()
educ = re.sub(r'[/ ]', '', educ)
categroy = html.xpath('//dd[@class="job_request"]/p/span[5]/text()')[0].strip()
job_request = html.xpath('//dd[@class="job_request"]/ul[@class="position-label clearfix"]//text()')
job_request = [i.strip() for i in job_request]
job_request = ' '.join(job_request)
publish_time = html.xpath('//dd[@class="job_request"]/p[@class="publish_time"]/text()')[0].strip()
publish_time = re.match(r'(.+)\xa0',publish_time).group(1)
job_advantage = html.xpath('//div[@class="content_l fl"]//dd[@class="job-advantage"]//text()')
job_advantage = [i.strip() for i in job_advantage]
job_advantage = ''.join(job_advantage)
job_bt = html.xpath('//dl[@id="job_detail"]/dd[@class="job_bt"]//text()')
job_bt = [i.strip() for i in job_bt]
job_bt = ''.join(job_bt)
work_addr = html.xpath('//div[@class="content_l fl"]//div[@class="work_addr"]//text()')
work_addr = [i.strip() for i in work_addr]
work_addr = work_addr[: -1]
work_addr = ''.join(work_addr)
company_name = html.xpath('//div[@class="content_r"]//img[@class="b2"]/@alt')[0].strip()
company_info = html.xpath('//div[@class="content_r"]//ul[@class="c_feature"]//text()')
company_info = [i.strip() for i in company_info]
company_info.reverse()
company_info = ''.join(company_info)
job = {
'department':department,
'job_name':job_name,
'salary':salary,
'city':city,
'work_year':work_year,
'educ':educ,
'categroy':categroy,
'job_request':job_request,
'publish_time':publish_time,
'job_advantage':job_advantage,
'job_bt':job_bt,
'work_addr':work_addr,
'company_name':company_name,
'company_info':company_info
}
return job
3-2. 直接分析ajax調(diào)用的接口,請(qǐng)求這個(gè)接口爬取數(shù)據(jù)
拉勾網(wǎng)的反爬蟲機(jī)制是基于cookies的株憾，同一個(gè)cookies僅可以重復(fù)幾次請(qǐng)求蝙寨，應(yīng)該是有時(shí)間方面的限制。嘗試使用代理IP也不行嗤瞎，可能cookies與IP進(jìn)行了綁定墙歪。且直接請(qǐng)求ajax接口的方式，很容易被發(fā)現(xiàn)是爬蟲而被封贝奇。下面的爬蟲代碼爬取兩三頁(yè)的數(shù)據(jù)便會(huì)被識(shí)別出來(lái)而被封虹菲。
def main():
sess = requests.session()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHT\
ML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
'Referer': 'https://www.lagou.com/jobs/list_Python?px=default&city=%E5%85%A8%E5%9B%BD',
'Host': 'www.lagou.com',
'Origin': 'https://www.lagou.com',
'X-Anit-Forge-Code': '0',
'X-Anit-Forge-Token': 'None',
'X-Requested-With': 'XMLHttpRequest'
}
data = {
'first': 'false',
'pn': '1',
'kd': 'python爬蟲'
}
url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'
jobs = [] #存儲(chǔ)所有職位信息
for page in range(1,26):
data['pn'] = page
response = sess.post(url=url, headers=headers, data=data)
print('獲取到的cookies：', sess.cookies.get_dict()) # 測(cè)試
page_html = response.text
# 獲取職位ID
positionIds = re.findall(r'"positionId":(\d{7})', page_html)
for positionId in positionIds:
try:
# 拼接成職位詳情頁(yè)鏈接
position_link = 'https://www.lagou.com/jobs/%s.html'%positionId
response = sess.get(url=position_link, headers=headers)
detail_page_html = response.text
job = parse_detail_position(detail_page_html)
jobs.append(job)
print(job) # 測(cè)試
except Exception:
print('錯(cuò)誤鏈接',position_link)
positionIds.append(positionId)
#隨機(jī)停頓幾秒
n = random.uniform(0.5, 3.5)
time.sleep(n)
return jobs
3-2. 使用selenium驅(qū)動(dòng)瀏覽器爬取數(shù)據(jù)
使用selenium+chromedriver就可以避開拉勾網(wǎng)的反爬機(jī)制。
def main():
driver = webdriver.Chrome()
url = 'https://www.lagou.com/zhaopin/Python/?labelWords=label'
driver.get(url)
# 存儲(chǔ)所有的職位信息
jobs = []
while True:
#直接通過WebElement對(duì)象獲取屬性(職位的詳情鏈接)
position_a = driver.find_elements_by_xpath('//a[@class="position_link"]')
position_links = [a.get_attribute('href') for a in position_a]
# 打開新窗口
driver.execute_script('window.open()')
# 切換到新窗口
driver.switch_to.window(driver.window_handles[-1])
for position_link in position_links:
#連續(xù)訪問詳情頁(yè)有時(shí)會(huì)返回異常頁(yè)面,導(dǎo)致解析報(bào)錯(cuò)
try:
driver.get(position_link)
page_html = driver.page_source
job = parse_detail_position(page_html)
jobs.append(job)
except Exception:
print('異常鏈接',position_link)
#錯(cuò)誤鏈接重新放回
position_links.append(position_link)
# 每詳情頁(yè)停頓
n = random.uniform(0.2, 1) # 均勻分布
time.sleep(n)
# break #測(cè)試
print(position_link) #測(cè)試
#關(guān)閉當(dāng)前詳情頁(yè)面
driver.close()
#切換列表頁(yè)窗口,注意:雖然詳情頁(yè)窗口已經(jīng)關(guān)閉,但仍然需要顯式的切換回列表頁(yè)窗口!selenium不會(huì)自動(dòng)切換窗口的掉瞳！
driver.switch_to.window(driver.window_handles[0])
#顯式等待;元素出現(xiàn)才獲取
next_page = WebDriverWait(driver,30).until(
EC.presence_of_element_located((By.XPATH, '//div[@class="pager_container"]/a[last()]'))
)
# 判斷是否到最后一頁(yè),此時(shí)下一頁(yè)不可點(diǎn)擊
if next_page.get_attribute('class') == 'page_no pager_next_disabled':
break
next_page.click() #點(diǎn)擊下一頁(yè)
#每列表頁(yè)停頓
n = random.uniform(0.5,2) #均勻分布
time.sleep(n)
break #測(cè)試
driver.quit() #關(guān)閉瀏覽器
return jobs
- 啟動(dòng)爬蟲毕源，并將數(shù)據(jù)寫入csv文件
if __name__ == '__main__':
jobs = main()
headers = ['department','job_name','salary','city','work_year','educ',\
'categroy','job_request','publish_time','job_advantage','job_bt',\
'work_addr','company_name','company_info']
with open('lagou.csv','w',encoding='utf-8',newline='') as fp:
writer = csv.DictWriter(fp,headers)
writer.writeheader()
writer.writerows(jobs)
