寫這個爬蟲的緣由
以前用java寫過一個爬取學(xué)校的教務(wù)系統(tǒng)的爬蟲 https://blog.csdn.net/ygdxt/article/details/81158321鞭光，最近癡迷Python爬蟲茬故，了解到許多強大的庫超营，想再一次用學(xué)校的教務(wù)系統(tǒng)做下測試实蓬。
這一次我首先想到的是新的教務(wù)系統(tǒng)赊抖，這個難度更大艘包，因為有了驗證碼識別反爬的猛，由于我是用的tessocr庫識別驗證碼,(具體配置過程可以參考我之前的博客 python填坑之路:tesserocr配置)
用Requests.get方法把驗證碼下載下來識別之后耀盗，同時因為我爬取網(wǎng)頁是用的selenium做的模擬網(wǎng)頁動作，這里就有一個同步性的問題卦尊，不能保證selenium請求網(wǎng)頁上的驗證碼和requests請求的驗證碼是同一個叛拷，相當(dāng)于selenium、requests分別請求了一次登陸網(wǎng)頁猫牡，兩個網(wǎng)頁上的驗證碼顯然是不同的胡诗。所以
怎么保證請求登錄界面得到的網(wǎng)頁上的驗證碼
和我們請求驗證碼服務(wù)器返回的驗證碼是同一個驗證碼是同一個是一個很迷人的問題，
我開始還以為可以從網(wǎng)頁源代碼上直接定位到這個驗證碼淌友，結(jié)果顯示這個驗證碼在登錄界面的
的存在形式不是一個..png/jpg,而是通過src=“驗證碼服務(wù)器”來實現(xiàn)異步加載
同時煌恢，由于tessocr識別驗證碼的成功率可能只有50%，要提高驗證率可能還要對接云打碼震庭，果斷放棄了爬取新教務(wù)系統(tǒng)的想法瑰抵，還是爬取原來的沒有驗證碼的舊教務(wù)系統(tǒng)，
其實新舊教務(wù)系統(tǒng)最大的區(qū)別就是登陸界面不一樣器联，登陸之后都一樣二汛，貌似用了重定向
ps:如果你對這個問題有什么好的解決辦法，請不吝賜教
運行結(jié)果預(yù)覽
20181128201749396.png
編碼過程
詳細(xì)的代碼解釋就看注釋吧拨拓，有什么問題歡迎交流
執(zhí)行爬蟲的主程序csu.py肴颊，里面有許多測試用的注釋代碼，就不刪了
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select
from config import *
import time
broswer = webdriver.Chrome()
wait = WebDriverWait(broswer, 10)
def search():
try:
broswer.get("http://csujwc.its.csu.edu.cn/jsxsd/kscj/yscjcx_list")
account = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#userAccount"))
)
password = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#userPassword"))
)
submit = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "#btnSubmit"))
)
except TimeoutException:
return search()
#登錄
account.send_keys(ACCOUNT)
password.send_keys(PASSWORD)
submit.click()
#進入我的成績界面
my_score = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR,"body > div.wap > a:nth-child(3) > div"))
)
my_score.click()
#成績和平均分
# my_rank = wait.until(
# EC.presence_of_element_located((By.CSS_SELECTOR, "#LeftMenu1_divChildMenu > ul > li:nth-child(4) > a"))
# )
# my_rank.click()
#
# rank = wait.until(
# EC.presence_of_element_located((By.CSS_SELECTOR, "#dataList > tbody > tr:nth-child(2) > td:nth-child(3)"))
# )
# #http://www.w3school.com.cn/cssref/selector_nth-child.asp nth-child(n)的用法
# average_score = wait.until(
# EC.presence_of_element_located((By.CSS_SELECTOR, "#dataList > tbody > tr:nth-child(2) > td:nth-child(4)"))
# )
#
# print('您的平均成績是:'+average_score.text+"\n排名:"+rank.text)
#逐次展示 我的成績八個子項
# css_selector = "#LeftMenu1_divChildMenu > ul > li:nth-child({0}) > a"
# for i in range(8):
# # 將滾動條移動到頁面的頂部
# js = "var q=document.documentElement.scrollTop=0"
# broswer.execute_script(js)
# time.sleep(2)
#
# aviable_score = wait.until(
# EC.presence_of_element_located((By.CSS_SELECTOR, css_selector.format(str(i+1))))
# )
# aviable_score.click()
#
#
# #將滾動條移動到頁面的底部
# for j in range(8):
# js="var q=document.documentElement.scrollTop="+str(j*200)
# broswer.execute_script(js)
# time.sleep(1)
#處理select https://www.cnblogs.com/imyalost/p/7846653.html
yxcj = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "#LeftMenu1_divChildMenu > ul > li:nth-child(1) > a"))
)
select_score_element = broswer.find_element_by_css_selector("#xnxq01id")
select_score = Select(select_score_element)
#得到下拉列表的所有子項
select_score_items = broswer.find_elements_by_css_selector("#xnxq01id option")
select_score_items_text = []
for item in select_score_items:
select_score_items_text.append(item.text)
#print(item.text)
scores_dic = {}
for i in range(len(select_score.options)):
#不加這兩行會報錯渣磷，原因： https://blog.csdn.net/ulebo/article/details/52128033
print("*****************************************************"+select_score_items_text[i]+
"*****************************************************")
select_score_element = broswer.find_element_by_css_selector("#xnxq01id")
select_score = Select(select_score_element)
select_score.select_by_index(i)
time.sleep(1)
score_table = broswer.find_element_by_css_selector("#dataList")
data = score_table.text.replace("+","")
data = data.split("\n")
datalist = []
for line in data:
datalist.append(line.split())
scores_dic[select_score_items_text[i]] = datalist
return scores_dic[select_score_items_text[0]]
def main():
search()
if __name__ =="__main__":
main()
ui.py程序的gui婿着，直接運行這個就好，它會調(diào)用csu.py
#coding=utf-8
import wx
import wx.grid
import csu
class UI(wx.Frame):
def __init__(self):
wx.Frame.__init__(self,parent=None,title="成績查詢",size=(1050,560))
grid = wx.grid.Grid(self,pos=(10,0),size=(1050,500))
grid.CreateGrid(100,9)
for i in range(100):
for j in range(9):
grid.SetCellAlignment(i,j,wx.ALIGN_CENTER,wx.ALIGN_CENTER)
grid.SetColLabelValue(0, "序號") #第一列標(biāo)簽
grid.SetColLabelValue(1, "初修學(xué)期")
grid.SetColLabelValue(2, "獲得學(xué)期")
grid.SetColLabelValue(3, "課程")
grid.SetColLabelValue(4, "成績") # 第一列標(biāo)簽
grid.SetColLabelValue(5, "學(xué)分")
grid.SetColLabelValue(6, "課程屬性")
grid.SetColLabelValue(7, "課程性質(zhì)")
grid.SetColLabelValue(8, "獲得方式") # 第一列標(biāo)簽
grid.SetColSize(0,50)
grid.SetColSize(1,100)
grid.SetColSize(2,100)
grid.SetColSize(3,350)
grid.SetColSize(4,50)
grid.SetColSize(5,50)
grid.SetColSize(6,50)
grid.SetColSize(7,100)
grid.SetColSize(8,100)
grid.SetCellTextColour("NAVY")
data = csu.search()
data.remove(data[0])
print(data)
for i,item1 in enumerate(data):
for j,item2 in enumerate(item1):
grid.SetCellValue(i,j,data[i][j])
pass
app = wx.App()
frame = UI()
frame.Show()
app.MainLoop()
