- 背景：剛好公司有一個刷數(shù)據(jù)的任務(wù)温艇，就上手了python，早該開始了幔戏。
- 小結(jié)知識點如下：
- 1 python的默認編碼是unicode啼器，如果在程序中輸入中文就會有亂碼旬渠，如果文件中中文是utf-8編碼的，就可以用以下這種方式解碼端壳。
import os
import codecs
path = os.getcwd();
print(path);
os.chdir('/Users/liuchaoqun01/Desktop')
print(os.getcwd());
with codecs.open('to see you', encoding='utf-8') as f:
print(f.read())
- 2 cc助攻的解碼問題：文本文件是latin1編碼的中文告丢，120w行。甚至蛋疼损谦，一直不知道應(yīng)該怎么轉(zhuǎn)為utf-8岖免。經(jīng)cc點化：
with open('d:/res.txt','wb') as w:
with open(r'd:/entity.list.all.txt','rb') as f:
for x in f:
try:
w.write(x.decode('gb2312').encode('utf8'))
except Exception:
w.write(b'\n')
還是一知半解，等待cc指正照捡！
- 3 這是最后成形的代碼颅湘，本質(zhì)上就是一個數(shù)據(jù)腳本，總共跑了1hour栗精，120w數(shù)據(jù)闯参，其實最后發(fā)現(xiàn)120w的數(shù)據(jù)查詢太耗時了，一開始就應(yīng)該用批量查詢１ⅰ鹿寨！
import sys,re,os
import datetime
import MySQLdb
begin = datetime.datetime.now()
reload(sys)
sys.setdefaultencoding("utf-8")
path = os.getcwd();
os.chdir('/Users/liuchaoqun01/Documents/cloud/input')
predictFile = open('predict.trade2.entity.list.all', 'r')
count = 0
businessList = []
emptyList = []
predictFileList = []
resultList = []
emptyList = []
with open('res.txt','r') as w:
for business in w:
line = business.replace('\n', '')
businessList.append(line)
for predict in predictFile:
p = re.compile(r'\s|__')
ss = p.split(predict)
if len(ss) < 3:
count = count + 1
line = '-1'
else:
line = "select industryname from aodfeed.industry where industryid = " + (ss[2])
predictFileList.append(line)
line = ''
predictFile.close()
print 'The count of blank line is:' + str(count)
# mysql
outputResult = open('/Users/liuchaoqun01/Documents/cloud/output/result', 'w+')
outputEmpty = open('/Users/liuchaoqun01/Documents/cloud/output/empty', 'w+')
conn = MySQLdb.connect(host='10.99.196.227', port=8306, user='test', passwd='xxx', db='xxx', charset='utf8')
cursor = conn.cursor()
for i in range(len(predictFileList)):
predict = predictFileList[i]
business = businessList[i]
if predict == '-1':
emptyList.append(business + '\t' + '-1')
else:
sql = predict
cursor.execute(sql)
r = cursor.fetchall()
# print (business + '\t' + r[0][0].decode('utf-8'))
resultList.append(business + '\t' + r[0][0])
# write
for out in resultList:
outputResult.write(out + '\n')
outputResult.close()
for out in emptyList:
outputEmpty.write(out + '\n')
outputEmpty.close()
print 'process end!'
end = datetime.datetime.now()
print (end - begin)
