成果
和之前的多進程相比黍析，這個實用點卖怜，原來那個并沒有處理好循環(huán)中的重復(fù)采集問題，實際上是不能用于多進程的
改動如下：
所有的讀寫操作都放在最后一步執(zhí)行阐枣，這樣能避免一些復(fù)雜的情況马靠，減少涉及的參數(shù)
讀取的東西化作數(shù)列或者會比較好
def read(kw,zm1s):
所有的讀寫操作只發(fā)生在final（kw）
記住return只能返回一次，然后函數(shù)就結(jié)束了
稍微引入了json
其他變化不大
import requests
import re
import time
import json
from multiprocessing import Pool
time1 = time.strftime("%H:%M:%S").replace(':','')
def get_baidu_dasou(keyword ='劍圣'):
try:
url1 = 'http://suggestion.baidu.com/su?json=1&p=3?wd=' + keyword
url2 = 'http://suggestion.baidu.com/su?sugmode=2&json=1&wd=' + keyword
reslt_baidu_dasou1 = requests.get(url1).text
reslt_baidu_dasou2 = requests.get(url2).text
reslt_baidu_dasou1 = reslt_baidu_dasou1.replace('window.baidu.sug(', '').replace(')', '').replace(';', '')
reslt_baidu_dasou2 = reslt_baidu_dasou2.replace('window.baidu.sug(', '').replace(')', '').replace(';', '')
jcont_reslt_baidu_dasou1 = json.loads(reslt_baidu_dasou1)
jcont_reslt_baidu_dasou2 = json.loads(reslt_baidu_dasou2)
final_longtaikeywords = set(jcont_reslt_baidu_dasou1['s']) | set(jcont_reslt_baidu_dasou2['s']) # 取集合交集
print(str(keyword)+':'+str(final_longtaikeywords))
return list(final_longtaikeywords) #返回的是數(shù)列
except Exception as e:
print(e)
time.sleep(5)
def get_sougou(keyword ='劍圣'):
final_longtaikeywords=[]
try:
urlsougou = 'https://www.sogou.com/suggnew/ajajjson?key='+ keyword+'&type=web&ori=yes&pr=web'
reslt_sougou_dasou1 = requests.get(urlsougou).text
reslt_sougou_dasou1=reslt_sougou_dasou1.split('[')[2] #獲得中間的長尾詞結(jié)果
kws = re.compile(r'"(.*?)",')
kws_list_dasou1 = kws.findall(reslt_sougou_dasou1)
for kws1 in kws_list_dasou1:
final_longtaikeywords.append(kws1)
print(str(keyword)+':'+str(final_longtaikeywords))
return (final_longtaikeywords) #返回的是數(shù)列
except Exception as e:
print(e)
time.sleep(5)
def get_360(keyword ='劍圣'):
final_longtaikeywords=[]
try:
url360 = 'http://sug.so.#/suggest?encodein=utf-8&encodeout=utf-8&format=json&fields=word,obdata&word=' + keyword
reslt_360_dasou1 = requests.get(url360).text
json_reslt_360 = json.loads(reslt_360_dasou1)
for i in range(0, len(json_reslt_360['result'])):
final_longtaikeywords.append(json_reslt_360['result'][i]['word'])
print(str(keyword) + ':' + str(final_longtaikeywords))
return (final_longtaikeywords) # 返回的是數(shù)列
except Exception as e:
print(e)
time.sleep(5)
def read(kw,zm1s):
final_longtais=[]
last_longtais=[]
for zm1 in zm1s:
print('百度搜索：'+kw+zm1)#是否還要判斷
final_longtais.append(get_baidu_dasou(kw+zm1))
print('百度搜索：' + zm1+ kw)
final_longtais.append(get_baidu_dasou(zm1+kw))#其他搜索引擎
print('搜狗搜索：' + kw + zm1)
final_longtais.append(get_sougou(kw + zm1))#搜狗不做
print('360搜索：' + kw + zm1)
final_longtais.append(get_360(kw + zm1))
print('360搜索：' + zm1+ kw)
final_longtais.append(get_360(zm1+ kw))
if final_longtais!=None:
for final_longtai in final_longtais: #第一層數(shù)列
if final_longtai != None:
for each_word in final_longtai: #數(shù)列中的數(shù)列
last_longtais.append(each_word)
last_longtais2=quchong(last_longtais)
print(last_longtais2)
print(len(last_longtais2)) #執(zhí)行完return就結(jié)束了甩鳄，所以這里只能返回一個數(shù)列
return last_longtais2
def quchong(firstlist): # 去重
last_firstlist = list(set(firstlist))
last_firstlist.sort(key=firstlist.index)
return last_firstlist
def final(kw):
resultpath= r'C:/Users/Administrator/Desktop/result/xiala/'+time1+'resultkeywords.txt'
finalpath = r'C:/Users/Administrator/Desktop/result/xiala/'+time1+'finalkeywords.txt'
lastpath = r'C:/Users/Administrator/Desktop/result/xiala/' +time1+'lastpath.txt'
zm1s=['',' ','a','b','c','d','e','f','g','h','j','k','l','m','n','o','p','q','r','s','t','w','x','y','z','0','1','2','3','4','5','6','7','8','9']
#zm1s = ['','a']
zm2s=['',' ']
#zm2s = ['']
finalist =[] # 擴展第二次,存放2層列表
finalist2= [] #存放最終的關(guān)鍵詞
lastlist=[] #擴展第三次
firstlist=read(kw,zm1s)#返回第一次排重后的數(shù)列
with open(resultpath,'a+',encoding='utf-8')as text:
for keywords in firstlist:
text.write(keywords+'\n')
lastlist=read(kw,zm1s)
if firstlist != None:
for kw in firstlist:
finalist.append(read(kw, zm2s)) #二層數(shù)列
if finalist != None:
for final_longtai in finalist: # 第一層數(shù)列
if final_longtai != None:
for each_word in final_longtai: # 數(shù)列中的數(shù)列
finalist2.append(each_word)
lastlist.append(each_word)
finalist2=quchong(finalist2)
with open(finalpath,'a+',encoding='utf-8')as text:
for keywords in finalist2:
text.write(keywords+'\n')
lastlist=quchong(lastlist)
with open(lastpath,'a+',encoding='utf-8')as text:
for keywords in lastlist:
text.write(keywords+'\n')
if __name__ == '__main__':#為什么會重復(fù)了
start = time.clock()
pool = Pool(processes=6)
keywords_list=[]
startpath1 = r'C:\Users\Administrator\Desktop/result\xiala\testforkeywords.txt' # 關(guān)鍵詞存放的地方
with open(startpath1, 'r', encoding='utf-8') as text: # 打開關(guān)鍵詞文件，做列表list
kwsline = text.readlines()
for kw in kwsline:
kw = kw.strip() # 去除空行
kw = kw.replace('\n', '') # 替換最后一行
keywords_list.append(kw)
pool.map(final,keywords_list)#這里似乎需要一次排重
pool.close()
pool.join()
end = time.clock()
print ("read: %f s" %(end - start))
