本想著一步到位懦冰，把bug都解決后再發(fā)布浪慌，無奈碰到個(gè)問題困擾了好幾天趁桃，就先發(fā)布下目前的版本吧仑最，做下階段備份扔役，代碼如下：
#-*- coding: utf-8 -*-
import urllib2
import re
import time
f=urllib2.urlopen('http://www.360kad.com/dymhh/allclass.shtml').read()
n1=f.find('<dt><span><a href="http://www.360kad.com/Category_47/Index.aspx" target="_blank">')
n2=f.find('4057/Index.aspx" target="_blank">')
n3=f[n1:(n2+90)]
n4=re.sub('<!--[\s\S]*?-->','',n3)
n5=re.findall('<dd>[\s\S]*?</dd>',n4)
#將鏈接和tag名做成列表模式
w1=[]#鏈接
e1=re.findall('http://www.360kad.com/Category_\d{1,}/Index.aspx',''.join(n5))
for i1 in e1:
w1.append(i1)
w2=[]#tag名
e2=re.findall('blank">\S{1,}?</a>',''.join(n5))
e3=(''.join(e2)).replace('blank">','').replace('</a>','\n')
for i2 in e3.split('\n'):
w2.append(i2)
w=[]#鏈接+tag名
for i in range(len(w2)-1):
w.append(w1[i])
w.append(w2[i])
n6=re.findall('_\d{1,}',''.join(n5))
kad_tag=[]
for n in n6:
kad_tag.append(n)
#每個(gè)
class Page_list:
def __init__(self):
self.url1='http://www.360kad.com/Category'
def request_open(self,n1,n2):
self.url2='/Index_'+str(n2+1)+'.aspx'
self.page1=urllib2.urlopen(self.url1+kad_tag[n1]+self.url2).read()
#print n2+1
return self.page1
#頁碼數(shù)
class Next_page:
def __init__(self):
self.url1='http://www.360kad.com/Category'
def np(self,n1):
self.page1=urllib2.urlopen(self.url1+kad_tag[n1]+'/Index.aspx').read()
self.url2=re.findall('_\d{1,}.aspx">尾頁</a>',self.page1)
#print self.url2
if self.url2:
self.num2=self.url2[-1][1:-17]
#print self.num2
return int(self.num2)
else:
return 1
page_list=Page_list()
page_text=[]
#具體每個(gè)頁面的產(chǎn)品名、規(guī)格词身、價(jià)格厅目、鏈接
class Page_info:
def __info__(self):
pass
def m(self,n1,n2):#產(chǎn)品名
self.page_name2=[]
self.page_name1=re.findall('title=".*?" rel="nofollow" class',page_list.request_open(n1,n2))
for x in range(len(self.page_name1)):
self.page_name2.append(self.page_name1[x][7:-22])
self.page_num2=[]#規(guī)格
self.page_num1=re.findall('"num">.*?<',page_list.request_open(n1,n2))
for i in range(len(self.page_num1)):
self.page_num2.append(self.page_num1[i][6:-2])
self.page_price2=[]#價(jià)格
self.page_price1=re.findall('"price">\S{1,}</span>|priceR">\S{1,}?<|<p class="vip_pric"[\s\S]{1,}</p>',page_list.request_open(n1,n2))
for s in self.page_price1:
self.p2=re.findall('style',s)
if self.p2:
self.page_price2.append('沒有價(jià)格')
else:
self.p3=re.findall('\d{1,}\.\d{1,}',s)
self.page_price2.append(self.p3[-1])
self.page_url2=[]#鏈接
self.page_url1=re.findall('<a class="name".*?title=',page_list.request_open(n1,n2))
self.u2=re.findall('/product/\d{1,}.shtml',''.join(self.page_url1))
for f in self.u2:
self.page_url2.append('http://www.360kad.com'+f)
for i in range(len(self.page_name2)):
self.dic=[]
self.dic.append(self.page_name2[i])
self.dic.append(self.page_num2[i])
self.dic.append(self.page_price2[i])
self.dic.append(self.page_url2[i])
page_text.append(self.dic)#這里有疑問
page_info=Page_info()
next_page=Next_page()
for x in range(len(w2)-375):#為減少工作量，就簡(jiǎn)單提取3個(gè)標(biāo)簽驗(yàn)證程序
print'正在抓取tag%s中的內(nèi)容'%(w2[x])
starttime2=time.time()
for i in range(next_page.np(x)):
print'開始抓取第%d頁法严，抓取進(jìn)度：'%(i+1)
starttim2=time.time()
page_info.m(x,i)#page_text.append(self.dic)會(huì)疊加
endtime2=time.time()
print'抓取第%d頁完畢损敷，用時(shí)%.2fs'%(i+1,endtime2-starttime2)
time.sleep(4)
'''w4=str(w2[x]).decode('utf-8')
t=file('%s.txt'%w4,'w')
#print type(page_text)
page_text0=''.join('%s' %id for id in page_text)
#page_text1=page_text0.decode('utf-8')
t.write(page_text0)
t.close()'''#tag保存的內(nèi)容會(huì)疊加，沒有搞清楚怎么解決
endtime2=time.time()
print '抓取%s標(biāo)簽完畢,用時(shí)%.2fs'%(w2[x],endtime2-starttime2)
f=file('KAD_ALL.html','w')
f.write('<!DOCTYPE html>\n<html>\n<head>\n')
f.write('<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n')
f.write('</head>\n\n<body>\n')
s=1
for i in page_text:
f.write('<p>'+str(s)+'. '+'<a href=\"'+i[3]+'\">'+i[0]+'</a>'+'深啤，規(guī)格：'+i[1]+'價(jià)格：'+i[2]+'\n')
s=s+1
f.write('</body>')
f.close()
print'抓取完成拗馒，請(qǐng)查看'
因?yàn)楣こ塘柯源螅韵敕謙ag保存溯街，避免后期出問題诱桂，之前的工作還得重新來。結(jié)果出現(xiàn)append內(nèi)容疊加的情況呈昔，就是說第二個(gè)tag保存的內(nèi)容內(nèi)也包含了之前的第一個(gè)tag的內(nèi)容挥等。
