可可英語四級備考界面
爬蟲基礎(chǔ)介紹：
- 1.
url:某個(gè)網(wǎng)頁的網(wǎng)址 - 2.帶反扒機(jī)制的網(wǎng)頁绘梦，加個(gè)
header
header={'User-Agent':'Mozilla/5.0
(Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/56.0.2924.87 Safari/537.36'}
- 3.模擬瀏覽器進(jìn)入網(wǎng)頁：
request = urllib2.Request(url,headers=header)
- 4.打開網(wǎng)址：
response = urllib2.urlopen(request)
- 5.獲取源碼漂问，讀取網(wǎng)頁
html = response.read()
- 6.編寫正則表達(dá)式：
pattern = re.compile(r"----正則表達(dá)式----")
- 7.匹配正則表達(dá)式：
items = re.findall(pattern,html)# 注意：此時(shí)的items是一個(gè)列表捂寿，用來存放匹配到的東西
- 代碼分享
#coding=utf-8
import urllib2
from constants import url
import re
import sys
import os
reload(sys)
sys.setdefaultencoding('utf-8')#解決編碼中出現(xiàn)亂碼問題
def get_title(url):
req = urllib2.Request(url)
req.add_header('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36')#添加頭部信息
res = urllib2.urlopen(req)
html = res.read()
response = re.compile(r'id="nrtitle">(.*?)</h1>', re.S)
title = re.findall(response, html)[0]#通過re模塊進(jìn)行內(nèi)容匹配查找
title = title.replace('?','')#字符串的替換
title = title.replace('&','')
title = title.replace(';','')
title = title.replace('#','')
title = title.replace('34','')
return title
def get_first_page(url):
req = urllib2.Request(url)
req.add_header('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36')
res = urllib2.urlopen(req)
html = res.read()
response = re.compile(r'<span id="article_eng">(.*?)<script>',re.S)
contents = re.findall(response,html)[0]
contents = contents.replace('</p>','')
contents = contents.replace('<p>','')
contents = contents.replace('<strong>','')
contents = contents.replace('</strong>','\n')
contents = contents.replace('<br />','')
return contents
def get_second_page(url):
# print '參考翻譯：'
req = urllib2.Request(url)
req.add_header('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36')
res = urllib2.urlopen(req)
html = res.read()
response = re.compile(r'<span id="article_eng">(.*?)<script>', re.S)
contents = re.findall(response, html)[0]
contents = contents.replace('</p>', '')
contents = contents.replace('<p>', '')
contents = contents.replace('<strong>', '')
contents = contents.replace('</strong>', '')
contents = contents.replace(''', '')
contents = contents.replace('<br />', '\n')
return contents
def get_third_page(url):
req = urllib2.Request(url)
req.add_header('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.109 Safari/537.36')
res = urllib2.urlopen(req)
html = res.read()
response = re.compile(r'<span id="article_eng">(.*?)<script>', re.S)
contents = re.findall(response, html)[0]
contents = contents.replace('</p>', '')
contents = contents.replace('<p>', '')
contents = contents.replace('<strong>', '')
contents = contents.replace('<br />', '')
contents = contents.replace('</strong>', '\n')
return contents
req = urllib2.Request(url)
res = urllib2.urlopen(req)
html = res.read()
reg = re.compile(r'http://www.kekenet.com/menu/201\d+/[0-9]{6}',re.S)
url_items = re.findall(reg,html)
fin_url=[]
for i in url_items:
urls = []
u1 = i+'.shtml'
u2 = i+'_2.shtml'
u3 = i+'_3.shtml'
urls.append(u1)
urls.append(u2)
urls.append(u3)
fin_url.append(urls)
print '開始。眯停。鲫构。'
for x in fin_url:
title = get_title(x[0])#函數(shù)的調(diào)用
# print title
first = get_first_page(x[0])
second = get_second_page(x[1])
third = get_third_page(x[2])
filename = u'%s.docx' %title.split(':')[1]#切片操作
filename = filename.decode('utf-8')
path = os.path.join('traslate12', filename)#python中路徑的拼接
f = open(path,'a')#打開文件
f.write('\t\t'+title+'\n'+'翻譯原文：\n'+first+'\n'+'譯文：\n'+second+'\n'+'詳細(xì)解釋\n'+third)#寫入處理后的數(shù)據(jù)
f.close()
print '爬蟲工作結(jié)束抵怎。奋救。。'
自動(dòng)化生成到文件中
word文檔實(shí)例展示
