Selenium庫
什么是Selenium庫
Selenium是自動(dòng)化測(cè)試工具吧黄，支持多種瀏覽器芜辕，爬蟲中主要用來解決JavaScript渲染的問題
可以用Selenium來完成網(wǎng)頁的下拉，點(diǎn)擊等等操作理郑”螅可以完全模擬瀏覽器加載網(wǎng)頁，點(diǎn)擊按鈕怨酝，就像人一樣。
如何安裝
pip3 install selenium
用法講解
- 基本使用 ----selenium驅(qū)動(dòng)chrome完成打開百度那先，輸入關(guān)鍵字农猬，點(diǎn)擊回車的動(dòng)作
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
browser = webdriver.Chrome(executable_path='/usr/bin/chromedriver', chrome_options=options)#打開瀏覽器,加了很多option，為了在命令行下可以正常調(diào)用Chrome
try:
browser.get('http://www.baidu.com')#打開url
input = browser.find_element_by_id('kw')#找到id為kw的元素售淡，并賦值給input
input.send_keys('Python')#input向這個(gè)元素傳入Python字符串
input.send_keys(Keys.ENTER)#向input這個(gè)元素傳入回車
wait = WebDriverWait(browser,10)#等待10秒
wait.until(EC.presence_of_element_located((By.ID,'content_left')))#一直等待斤葱，直到id= content_left的元素加載完畢
print(browser.current_url)#打印當(dāng)前的url
print(browser.get_cookies())#打印當(dāng)前瀏覽器的cookie
print(browser.page_source)#打印當(dāng)前頁面的源代碼
finally:
browser.close()#關(guān)閉瀏覽器
-
聲明瀏覽器對(duì)象
from selenium import webdriver
browser = webdriver.Chrome(executable_path='/usr/bin/chromedriver', chrome_options=options)
browser = webdriver.Firefox()
browser = webdriver.Edge()
-
訪問頁面
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
browser = webdriver.Chrome(executable_path='/usr/bin/chromedriver', chrome_options=options)
try:
browser.get('http://www.taobao.com')#訪問淘寶
print(browser.page_source)#返回頁面源代碼
finally:
browser.close()
-
查找元素，比如找到input 輸入信息揖闸，找到button揍堕，模擬點(diǎn)擊
查找單個(gè)元素
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
browser = webdriver.Chrome(executable_path='/usr/bin/chromedriver', chrome_options=options)
try:
browser.get('http://www.w3school.com.cn/css/index.asp')
input_first = browser.find_element_by_id('intro')
input_second = browser.find_element_by_css_selector('#intro')
input_third = browser.find_element_by_xpath('//*[@id="intro"]')
print(input_first,input_second,input_third)
finally:
browser.close()
返回結(jié)果:
<selenium.webdriver.remote.webelement.WebElement (session="c29059732801b21c6317ce92c7484751", element="0.057789670625651324-1")> <selenium.webdriver.remote.webelement.WebElement (session="c29059732801b21c6317ce92c7484751", element="0.057789670625651324-1")> <selenium.webdriver.remote.webelement.WebElement (session="c29059732801b21c6317ce92c7484751", element="0.057789670625651324-1")>
有以下查找元素的方法
- find_element_by_name
- find_element_by_xpath
- find_element_by_link_text
- find_element_by_partial_link_text
- find_element_by_tag_name
- find_element_by_class_name
- find_element_by_css_selector
- 通用查找模式 find_element(By.ID,'intro')
查找多個(gè)元素
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
browser = webdriver.Chrome(executable_path='/usr/bin/chromedriver', chrome_options=options)
try:
browser.get('http://www.taobao.com')
lis = browser.find_elements_by_css_selector('.service-bd li')#注意是find_elements_XXX
lis = browser.find_elements(By.CSS_SELECTOR,'.service-bd li')#和上面的方法的結(jié)果一樣。這里用通用查找多個(gè)元素方法
print(lis)#返回一個(gè)列表
finally:
browser.close()
有多種方法查找多個(gè)元素
- find_elements_by_name
- find_elements_by_xpath
- find_elements_by_link_text
- find_elements_by_partial_link_text
- find_elements_by_tag_name
- find_elements_by_class_name
- find_elements_by_css_selector
- 通用查找模式 find_elements(By.ID,'intro')
元素交互操作
對(duì)獲取的元素調(diào)用交互方法
我們用上面的find_element等方法的到了元素汤纸，那么怎么操作這些元素呢衩茸？
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
import time
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
browser = webdriver.Chrome(executable_path='/usr/bin/chromedriver', chrome_options=options)
try:
browser.get('http://www.taobao.com')
input = browser.find_element_by_css_selector('#q')#找到id='q'的input
input.send_keys("iPhone")#在這個(gè)input中輸入iPhone
time.sleep(1)#等待1秒鐘
input.clear()#清空輸入框
input.send_keys("iPad")#input輸入iPad
button = browser.find_element_by_class_name('btn-search')#找到類名為‘btn-search’的元素
button.click()#這個(gè)元素執(zhí)行click函數(shù)
finally:
browser.close()
交互動(dòng)作 與元素交互動(dòng)作不同
將動(dòng)作附加到動(dòng)作鏈中串行執(zhí)行
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
from selenium.webdriver import ActionChains
options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--headless')
options.add_argument('--disable-gpu')
browser = webdriver.Chrome(executable_path='/usr/bin/chromedriver', chrome_options=options)
try:
browser.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
browser.switch_to.frame('iframeResult')
source = browser.find_element_by_css_selector('#draggable')
target = browser.find_element_by_css_selector('#droppable')
actions = ActionChains(browser)
actions.drag_and_drop(source,target)
actions.perform()
finally:
browser.close()
執(zhí)行JavaScript
from selenium import webdriver
browser = webdirver.Chrome()
browser.get('http://www.zhihu.com/explore')
browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
browser.execute_script('alert("To Bottom")')
獲取元素信息
獲取屬性
from selenium import webdriver
from selenium.webdriver import ActionChains
browser = webdriver.Chrome()
url = 'https://www.zhihu.com/explore'
browser.get(url)
logo = browser.find_element_by_id('zh-top-link-logo')
print(logo)
print(logo.get_attribute('class'))
獲取文本值
from selenium import webdriver
browser = webdriver.Chrome()
url = 'http://www.zhihu.com/explore'
browser.get(url)
input = browser.find_element_by_class_name('zu-top-add-question')
print(input.text)
獲取ID、位置贮泞、標(biāo)簽名楞慈、size
from selenium import webdriver
browser = webdriver.Chrome()
url = 'http://www.zhihu.com/explore'
browser.get(url)
input = browser.find_element_by_class_name('zu-top-add-question')
print(input.id)
print(input.location)
print(input.tag_name)
print(input.size)
Frame
網(wǎng)頁可能存在不止一個(gè)frame
import time
from selenium import webdirver
from selenium.common.exceptions import NoSuchElementException
browser = webdriver.Chrome
url = 'http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable'
browser.get(url)
browser.swithch_to.frame('iframeResult')
print(source)
try:
logo = browser.find_element_by_class_name('logo')
except NoSuchElementException:
print('NO LOGO')#子frame找不到父frame的元素
browser.switch_to.parent_frame()
logo = browser.find_element_by_class_name('logo')
print(logo)
print(logo.text)
等待
隱式等待
當(dāng)使用了隱式等待執(zhí)行測(cè)試的時(shí)候幔烛，如果WebDriver沒有在DOM中找到元素，將繼續(xù)等待囊蓝，超出設(shè)定時(shí)間后則拋出找不到元素的異常饿悬，換句話說，當(dāng)查找元素或元素并沒有立即出現(xiàn)的時(shí)候聚霜，隱式等待將等待一段時(shí)間在查找DOM狡恬，默認(rèn)的時(shí)間是0
from selenium import webdirver
browser = webdirver.Chrome()
browser.implicity_wait(10)#設(shè)置隱式等待10秒，意義：如果知乎網(wǎng)站沒有打開蝎宇，那么默認(rèn)等待10秒傲宜，如果10秒之后還找不到元素，再拋出異常夫啊。如果網(wǎng)速特別慢函卒，可以用這個(gè)辦法
browser.get('https://www.zhihu.com/explore')
input = browser.find_element_by_class_name('zu-top=add-question')
print(input)
顯式等待 非常常用
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
wait = WebDriverWait(browser,10)#顯式等待，更加常見撇眯，設(shè)置最長等待時(shí)間10秒
input = wait.until(EC.presence_of_element_located((By.ID,'q')))#等待條件是如果獲取id是q的元素
button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'btn-search')))#等待條件是獲取class='btn-search'的按鈕
print(input,button)
以下是expected_conditions常見操作
- title_is 標(biāo)題是某內(nèi)容
- title_contains 標(biāo)題包含某內(nèi)容
- presence_of_element_located 元素加載出报嵌，傳入定位元組，如（By.ID,'p'）
- visibility_of_element_located 元素可見熊榛，傳入定位元組
- visibility_of 可見锚国，傳入元素對(duì)象
- presence_of_all_elements_located 所有元素加載出
- text_to_be_present_in_element 某個(gè)元素文本包含某文字
- text_to_be_present_in_element_value 某個(gè)元素值包含某文字
- frame_to_be_available_and_switch_to_it frame加載并切換
- invisibility_of_element_located 元素不可見
- element_to_be_cliable 元素可點(diǎn)擊
- staleness_of 判斷一個(gè)元素是否人在DOM，可判斷頁面是否已經(jīng)刷新
- element_to_be_selected 元素可選擇玄坦，傳元素對(duì)象
- element_located_to_be_selected 元素可選擇血筑，傳入定位元組
- element_selection_state_to_be 傳入元素對(duì)象以及狀態(tài)，相等返回True煎楣，否則返回False
- element_located_selection_state_to_be 傳入定位元組以及狀態(tài)豺总，相等返回True，否則返回False
- alert_is_present 是否出現(xiàn)Alert
詳細(xì)內(nèi)容
瀏覽器的前進(jìn)后退
import time
from selenium import webdriver
browser = webdriver.Chrome()
browser.get('http://www.baidu.com')
browser.get('http://www.taobao.com')
browser.get('http://www.python.com')
browser.back()#后退
time.sleep(1)
browser.forward()#前進(jìn)
browser.close()
Cookies择懂，如果做登陸后爬取非常有用
from selenium import webdirver
browser = webdriver.Chrome()
browser.get('https://www.zhihu.com/explore')
print(browser.get_cookies())
browser.add_cookie({'name':'name','domain':'www.zhihu.com','value':'hello'})
print(browser.get_cookies())
browser.delete_all_cookies()
print(browser.get_cookies())
選項(xiàng)卡管理
import time
from selenium import webdriver
browser = webdirver.Chrome()
browser.get('https://www.baidu.com')
browser.execute_script('window.open()')#新增選項(xiàng)卡
print(browser.window_handles)#返回瀏覽器窗口句柄
browser.switch_to_window(browser.window_handles[1])#切換到第二個(gè)選項(xiàng)卡
browser.get('https://www.taobao.com')#第二個(gè)選項(xiàng)卡打開淘寶
time.sleep(1)
browser.switch_to_window(browser.window_handles[0])#切換到第一個(gè)選項(xiàng)卡
browser.get('http://python.org')#第一個(gè)選項(xiàng)卡打開python.org
異常處理
from selenium import webdriver
browser = webdriver.Chrome()
browser.get('https://www.baidu.com')
browser.find_element_by_id('hello')#不存在id=hello的元素喻喳，拋出異常NoSuchElementException
