# coding=utf-8
'''
download_html:接受url钝凶，返回html和BeautifulSoup實例spider接受html返回url和數(shù)據(jù)process_data:處理字符串及保存數(shù)據(jù)controller:控制，調(diào)用'''
__author__ ='Leslie'
from urllib.requestimport urlopen
from bs4import BeautifulSoup
import re,collections,os
# 接受url灯萍，返回html和BeautifulSoup實例
def download_html(url):
html = urlopen(url).read().decode('utf-8')#獲取頁面數(shù)據(jù)
? ? soup = BeautifulSoup(html,'lxml')#實例化BeautifulSoup
? ? return html,soup
#spider接受html返回url隊列钉嘹、title隊列问畅、數(shù)據(jù)
def spider(html=False,soup=False):
# 爬取首頁的url和title
? ? if not htmland soup:
queue_url = collections.deque()# 存儲url隊列
? ? ? ? queue_title = collections.deque()# 存儲標(biāo)題隊列
? ? ? ? # 定位元素罕邀，提取a標(biāo)簽href和title屬性
? ? ? ? for itemin soup.find_all("div", {"class":"box"}):
for Alabelin item.find_all("a"):
queue_url.append(Alabel["href"])
# 處理title字符串中多余的字符
? ? ? ? ? ? ? ? Str1 = Alabel["title"]
Str2 ='_盜墓筆記9在線閱讀_盜墓筆記全集'
? ? ? ? ? ? ? ? if Str2in Str1:
Str1 = Str1.replace(Str2,'')
index = Str1.index(']')
Str1 = Str1[index +1:].strip()
queue_title.append(Str1)
return queue_url,queue_title
# 爬取文字
? ? if htmland soup:
all_p_label = soup.find("div",class_="content-body").find_all("p")
return all_p_label
# 處理字符串及保存數(shù)據(jù)
def process_data(Data,title):
# 標(biāo)題名去除不可用字符[\/?:*<>"|]
? ? while '\\' in title:
index = title.index('\\')
title = title[:index] + title[index +1:]
matchList = re.findall('[/?:*<>"|]*', title)
matchStr =''.join(matchList)# '?><'
? ? title =list(title)
for jin matchStr:
title.remove(j)
title =''.join(title)
#保存文件的絕對路徑
? ? abspath = os.path.join(os.path.abspath(r'.\daomubiji1'), title)
#去除文字中多余的字符串如：www.setupu.com
? ? CMP = re.compile("(http://)?([a-zA-Z]+\.)+com")# 編譯正則表達(dá)式對象
? ? for iin Data:
each_string =str(i.string).replace(" ","").strip()
if each_string !="None":
Match = re.search(CMP, each_string)# 匹配字符串
? ? ? ? ? ? # 保存文字到txt文件
? ? ? ? ? ? with open(abspath,'a',encoding='utf-8')as fp:
if Match !=None:
Newstring = each_string[:Match.span()[0]]
fp.write(Newstring +'\n')
else:
fp.write(each_string +'\n')
# 控制乏苦，調(diào)用
def controller(url):
# 獲取要爬取的url隊列和文件名標(biāo)題
? ? html,soup = download_html(url)
queue_url,queue_title = spider(soup=soup)
# 循環(huán)爬取url知道隊列為空
? ? while url:
url = queue_url.popleft()
title = queue_title.popleft() +'.txt'
? ? ? ? print(title,url)
html,soup = download_html(url)
text_data = spider(html,soup)
process_data(text_data,title)
url =r'http://www.seputu.com/'
os.mkdir(os.path.abspath(r'.\daomubiji1'))
controller(url)
