這是學(xué)習(xí)python第二天，
由于有項目需要尼变，必須開始快速弄
頭一天大概看了下語法利凑、python爬蟲的視頻，決定今天開整
首先，讀一下別人的代碼截碴，自己給加上注釋
#引入包
import json
from multiprocessing import Pool
import requests
from requests.exceptions import RequestException
import re
#獲取頁面梳侨，并返回頁面代碼
def get_one_page(url):
try:
response = requests.get(url)
if response.status_code == 200: #訪問成功
return response.text
return None
except RequestException:
return None
#使用正則表達(dá)式解析頁面
def parse_one_page(html):
pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
+'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
+'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S)
items = re.findall(pattern, html)
for item in items:
yield {
'index': item[0],
'image': item[1],
'title': item[2],
'actor': item[3].strip()[3:],
'time': item[4].strip()[5:],
'score': item[5]+item[6]
}
#寫入文件
def write_to_file(content):
with open('result.txt', 'a', encoding='utf-8') as f:
f.write(json.dumps(content, ensure_ascii=False) + '\n')
f.close()
def main(offset):
url = 'http://maoyan.com/board/4?offset=' + str(offset)
html = get_one_page(url)
for item in parse_one_page(html):
print(item)
write_to_file(item)
if __name__ == '__main__':
pool = Pool()
pool.map(main, [i*10 for i in range(10)])
pool.close()
pool.join()
'''
https://www.zhihu.com/question/49136398 這個網(wǎng)址對此語句有詳細(xì)解釋和程序說明
注意if __name__ == '__main__'這一行，當(dāng)模塊從import當(dāng)中加載的時候這行保證下面的代碼不會執(zhí)行日丹。
'''
今天要做的就是抓取頁面的序列頁面走哺，和再抓取下一層的頁面詳細(xì)內(nèi)容
#引入包
import json
from multiprocessing import Pool
import requests
from requests.exceptions import RequestException
import re
import pymysql
host = '127.0.0.1'
username = 'root'
password = '123456'
database = 'heldum'
#測試數(shù)據(jù)庫連接
def testconnect():
#打開數(shù)據(jù)庫鏈接
db = pymysql.connect(host,username,password,database)
#使用cursor() 方法創(chuàng)建一個游標(biāo)對象 cursor
cursor = db.cursor()
#使用execute()方法執(zhí)行SQL查詢
cursor.execute("select version()")
#使用fetchone ()獲取單條數(shù)據(jù)
data = cursor.fetchone()
print(data)
db.close()
def InsertData(sql,item):
#打開數(shù)據(jù)庫鏈接
db = pymysql.connect(host,username,password,database,charset='utf8')
#使用cursor() 方法創(chuàng)建一個游標(biāo)對象 cursor
cursor = db.cursor()
remark = "測試插入信息"
#Sql 插入語句
#sql = "insert into gtzy(lm) " + "VALUES ('aaaa')"
try:
#執(zhí)行sql
#print("執(zhí)行插入")
tt = cursor.execute(sql,(item.__getitem__('url'),item.__getitem__('lm'),item.__getitem__('title'),item.__getitem__('date'),parse_one_page_content(get_one_page(item.__getitem__('url')))))
db.commit()
except UnicodeEncodeError as e :
#發(fā)生錯誤時回滾
print(e)
db.rollback()
db.close()
#加入headers
headers = { "Accept":"text/html,application/xhtml+xml,application/xml;",
"Accept-Encoding":"gzip",
"Accept-Language":"zh-CN,zh;q=0.8",
"Referer":"http://www.example.com/",
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36" }
#先寫主函數(shù)
def main(page_num):
url = 'url?key=%&type=0&page=' + str(page_num)
html = get_one_page(url)
for item in parse_one_page(html):
print(str(page_num))
InsertData("insert into gtzy(url,lm,title,date,content) VALUES (%s,%s,%s,%s,%s)",item)
#write_to_file(item)
#獲取主要內(nèi)容頁面內(nèi)容
def parse_one_page_content(html):
pattern = re.compile(' ', re.S)
items = re.findall(pattern, html)
dr = re.compile(r'<[^>]+>', re.S)
dd = dr.sub('', items[0])
return dd
#獲取頁面，并返回頁面代碼
def get_one_page(url):
try:
response = requests.get(url,headers=headers)
if response.status_code == 200: #訪問成功
return response.text
return None
except RequestException:
return None
'''
遇到403服務(wù)器拒絕爬蟲的服務(wù)協(xié)議怎么辦哲虾，要加入headers
'''
#使用正則表達(dá)式解析頁面
def parse_one_page(html):
pattern = re.compile('', re.S)
items = re.findall(pattern, html)
#print(items)
for item in items:
yield {
'url': item[0],
'title': item[1],
'lm': item[2],
'date': item[3]
}
#寫入文件
def write_to_file(content):
with open('result.txt', 'a', encoding='utf-8') as f:
f.write(json.dumps(content, ensure_ascii=False) + '\n')
f.close()
if __name__ == '__main__':
#InsertDate("insert into gtzy(url,lm,title,date,content) " +"VALUES ('"+username+"','aaaa','aaaa','aaaa','aaaa')")
pool = Pool()
pool.map(main, [i-1 for i in range(5891)])
pool.close()
pool.join()
'''
https://www.zhihu.com/question/49136398 這個網(wǎng)址對此語句有詳細(xì)解釋和程序說明
注意if __name__ == '__main__'這一行丙躏，當(dāng)模塊從import當(dāng)中加載的時候這行保證下面的代碼不會執(zhí)行。
'''
最后成功解決了
