進(jìn)一步優(yōu)化

1. 使用phantomjs替代firefox，實現(xiàn)無瀏覽器界面，加快速度
2. 去除上版對于瀏覽器布局部分代碼

優(yōu)化后的結(jié)果

基本上可以在5分鐘內(nèi)搞定上海的股票數(shù)據(jù)下載比過去確實有大進(jìn)步

#coding=utf-8
from selenium import webdriver
import time
import os
import re
import sys  
import threading
import Queue
from selenium.common.exceptions import NoSuchElementException
def myinit():
    reload(sys)  
    sys.setdefaultencoding('utf8')

########################
#生成多個瀏覽器實例
def makebr(number):
    brs=[]
    for i in range(number):
        #br=webdriver.Firefox()
        br=webdriver.PhantomJS('phantomjs')
        brs.append(br)
    return brs

#關(guān)閉所有瀏覽器
def closebr(brs):
    for i in brs:
        i.quit()
#######################
#獲得指定股票相關(guān)數(shù)據(jù)
def getinfo(mydriver,gourl):
    title='//*[@id="name"]'
    code='//*[@id="code"]'
    hexinshujuxpath="/html/body/div[14]/div[1]/div[4]/div[1]"
    restr="：".decode('utf8')
    myre=re.compile(restr,re.I|re.M|re.S)
    linetext=""
    errorinfo=""
    mydriver.get(gourl)
    try:
        gupiaoming=mydriver.find_element_by_xpath(title).text
        gupiaocode=mydriver.find_element_by_xpath(code).text
        hexinshuju=mydriver.find_element_by_class_name('pad5')
        shujuhang=hexinshuju.find_elements_by_tag_name('tr')
        for i in range(len(shujuhang)-2):
            shujulie=shujuhang[i].find_elements_by_tag_name('td')
            tmpshuju=myre.split(shujulie[0].text)
            linetext=linetext+"~"+tmpshuju[1]
        shuju=myre.split(shujuhang[8].text)
        linetext=linetext+"~"+shuju[1]
        tmpshuju=myre.split(shujuhang[9].text)
        linetext=linetext+"~"+tmpshuju[1]
        linetext="%s~%s%s"%(gupiaoming,gupiaocode,linetext)
        #print "數(shù)據(jù)：",linetext
    except  NoSuchElementException,e:
        #print "不是股票"
        pass
    except Exception:
        errorinfo= "非預(yù)期錯誤"+gourl
        print errorinfo
    finally:
        return linetext,errorinfo

#獲得所有股票鏈接
def geturls(br):
    
    #通過link對象獲得鏈接地址的text文本
    def getlinkurl(linklist):
        my=[]
        for x in linklist:
            my.append(x.get_attribute('href'))
        return my
    sz=[]
    sh=[]
    br.get("http://quote.eastmoney.com/stocklist.html")
    shxpath="/html/body/div[9]/div[2]/div/ul[1]"
    szxpath="/html/body/div[9]/div[2]/div/ul[2]"
    shgupiao=br.find_element_by_xpath(shxpath)
    szgupiao=br.find_element_by_xpath(szxpath)
    shgupiaolist=shgupiao.find_elements_by_tag_name('a')
    szgupiaolist=szgupiao.find_elements_by_tag_name('a')
    sh=getlinkurl(shgupiaolist)
    sz=getlinkurl(szgupiaolist)
    return sh,sz

#多線程執(zhí)行用的函數(shù)
def thread_getinfo(br,jobslink_queue,jieguo_queue,errorinfo_queue):
    while True:
        try:
            #獲得隊列里的地址
            url=jobslink_queue.get(False)  #False =Don't wait
        except Queue.Empty:
            print "完成退出"
            #br.quit()
            return
        #print url
        if(url!=None):
            (linetext,errorinfo)=getinfo(br,url)
            if(linetext!=""):
                jieguo_queue.put(linetext)
            if(errorinfo!=""):
                errorinfo_queue.put(errorinfo)
                

#######################
#多線程控制函數(shù)
def saveinfoabc(info_filename,error_filename,urllist):

    jobslink=Queue.Queue(0)
    jieguo=Queue.Queue(0)
    errorsinfo=Queue.Queue(0)
    
    for x in urllist[200:250]:#為測試方便這里只取了50俭茧，如果要全下載，取消[]就好
    #for x in urllist:
        jobslink.put(x)
    
    #啟動線程
    for x in range(THREAD_NUM):
        t=threading.Thread(target=thread_getinfo,args=(brs[x],jobslink,jieguo,errorsinfo))
        t.start()
    
    f=open(info_filename,'w')
    e=open(error_filename,'w')
    mycount=0
    
    while (threading.activeCount()>1) or (not jobslink.empty()):
        while jieguo.qsize()>0 or errorsinfo.qsize()>0:
            if(jieguo.qsize()>0):
                jieguotxt=jieguo.get()
                f.write(jieguotxt+"\n")
            if(errorsinfo.qsize()>0):
                error=errorsinfo.get()
                e.write(error+"\n")
        mycount=mycount+1
        if(mycount%100)==0:
            print "%d:  活動線程：%d,剩余連接數(shù)：%d,結(jié)果剩余條數(shù)：%d,錯誤剩余條數(shù):%d"%(mycount,threading.activeCount(),jobslink.qsize(),jieguo.qsize(),errorsinfo.qsize())
        time.sleep(0.01)
    e.close()
    f.close()
    
    print "數(shù)據(jù)下載完成"

######################
myinit()

#br=webdriver.Firefox()
br=webdriver.PhantomJS('phantomjs')
print "獲得所有鏈接地址"
(sh,sz)=geturls(br)
br.quit()

#info_filename='shinfo.txt'
#error_filename='sherror.txt'
THREAD_NUM=10
brs=makebr(THREAD_NUM)

saveinfoabc('shinfo_for_008.txt','sherror_for_008.txt',sh)
saveinfoabc('szinfo.txt','szerror.txt',sz)
closebr(brs)