前言

最近在之前的數(shù)據(jù)基本預(yù)處理工作之上功蜓，增加了兩個(gè)新的工作宠蚂。

1.數(shù)據(jù)擴(kuò)容，將數(shù)據(jù)的維度由3列擴(kuò)容到13列

2.對(duì)于數(shù)據(jù)擴(kuò)容后的數(shù)據(jù)求厕，進(jìn)行兩方面的條件篩選：
a.通過(guò)pid呀癣、起止時(shí)間查詢(xún) getInfoBypidtime(方法名)
b.通過(guò)pid、星期项栏、規(guī)定時(shí)段查詢(xún) getInfoBypid_wk_trange(方法名)

數(shù)據(jù)擴(kuò)容

之前規(guī)整后的數(shù)據(jù)有三列
數(shù)據(jù)格式如下：

2017-08-28 23:37:00,14,84742aa602e4

現(xiàn)在想對(duì)數(shù)據(jù)進(jìn)行擴(kuò)容處理沼沈，增加列數(shù)，以便于后續(xù)更加復(fù)雜的數(shù)據(jù)處理

設(shè)定各個(gè)列的屬性為：

[原數(shù)據(jù),時(shí)間戳,年,月,日,時(shí),分,秒,星期,分秒,pid,mac,是否為節(jié)假日]

擬得到的數(shù)據(jù)形式為：

2017-08-28 23:37:00,1503934620,2017,8,28,23,37,0,0,2337,14,84742aa602e4,N

數(shù)據(jù)擴(kuò)容 python 代碼如下：

# -*- coding: UTF-8 -*-

import csv
import numpy as np
import pandas as pd
from pandas import Series, DataFrame
from dateutil.parser import parse
import datetime
import time

__author__ = 'SuZibo'

"""
增加打散數(shù)據(jù)的維度
原數(shù)據(jù),時(shí)間戳,年,月,日,時(shí),分,秒,星期,分秒,pid,mac,是否為節(jié)假日
"""


holiday =['2017-09-02','2017-09-03','2017-09-09','2017-09-10','2017-09-16','2017-09-17','2017-09-23','2017-09-24','2017-10-01','2017-10-02','2017-10-03','2017-10-04','2017-10-05','2017-10-06','2017-10-07','2017-10-08','2017-10-14','2017-10-15','2017-10-21','2017-10-22','2017-10-28','2017-10-29','2017-11-04','2017-11-05','2017-11-11','2017-11-12']
#節(jié)假日數(shù)組芽腾，字符串類(lèi)型

rs = open('./macdata/normalinfo_add.txt', 'w')
#輸出文件為：normalinfo_add.txt

with open('./macdata/normal_origin_info.txt') as file:
#打開(kāi)文件

    for line in file:

        new_line = []
        #將每行對(duì)應(yīng)的原始數(shù)據(jù)擴(kuò)容后的數(shù)據(jù)页衙，每項(xiàng)寫(xiě)入新列表
        line = line.split(',')
        new_line.append(line[0])
        #保留源數(shù)據(jù)（第一列）

        timeArray = time.strptime(line[0], "%Y-%m-%d %H:%M:%S")
        #生成時(shí)間數(shù)組

        hr_min = str(timeArray.tm_hour) + str(timeArray.tm_min)

        new_line.append(int(time.mktime(timeArray)))
        #添加時(shí)間戳
        new_line.append(timeArray.tm_year)
        #年份
        new_line.append(timeArray.tm_mon)
        #月份
        new_line.append(timeArray.tm_mday)
        #日期
        new_line.append(timeArray.tm_hour)
        #小時(shí)
        new_line.append(timeArray.tm_min)
        #分鐘
        new_line.append(timeArray.tm_sec)
        #seconds
        new_line.append(timeArray.tm_wday)
        #星期幾 0-6 0是周一
        new_line.append(hr_min)
        #小時(shí)+分鐘  format:2337

        new_line.append(line[1])
        #地點(diǎn)id
        new_line.append(line[2])
        #mac info

        if  line[0][:10] in holiday:
            new_line.append('Y')
        else:
            new_line.append('N')
        #是否是節(jié)假日

        rs.write(str(new_line[0])+','+str(new_line[1])+','+str(new_line[2])+','+str(new_line[3])+','+str(new_line[4])+','+str(new_line[5])+','+str(new_line[6])+','+str(new_line[7])+','+str(new_line[8])+','+str(new_line[9])+','+str(new_line[10])+','+str(new_line[11].strip('\n'))+','+str(new_line[12])+'\n')
#第12位mac地址后面自帶換行符拷姿，要去掉
#最后+'\n'是為了文件換行

rs.close()

# 2017-08-28 23:37:00,14,84742aa602e4
# 2017-08-28 23:37:00,ts,2017,08,28,23,37,00,1,2337,14,84742aa602e4

數(shù)據(jù)篩選

好了，數(shù)據(jù)擴(kuò)容（數(shù)據(jù)進(jìn)一步規(guī)整化了）之后描滔，要對(duì)得到的數(shù)據(jù)進(jìn)行進(jìn)一步的篩選
這里得自己寫(xiě)篩選條件
方法1：getInfoBypidtime(pid,start_time,end_time)
參數(shù)說(shuō)明：

pid:地點(diǎn)id
start_time:查詢(xún)開(kāi)始時(shí)間踪古，格式為：'2017-08-28'
end_time:查詢(xún)結(jié)束時(shí)間，格式為：'2017-08-28'

方法2：getInfoBypid_wk_trange(idplaces,weekday,timeranges)
參數(shù)說(shuō)明：

pid:地點(diǎn)集合拘泞，格式為:[151,152,153]
weekday:星期幾的集合枕扫，格式為:[2,3,4]
timeranges:[[830,1200],[1400,1830],......]

python代碼如下：

# -*- coding: UTF-8 -*-

import numpy as np
import pandas as pd
from pandas import Series, DataFrame
from dateutil.parser import parse
import datetime

__author__ = 'SuZibo'

"""
查詢(xún)數(shù)據(jù)
1.通過(guò)pid、起止時(shí)間查詢(xún)
2.通過(guò)pid、星期染簇、規(guī)定時(shí)段查詢(xún)
"""

# records = pd.read_csv('./macdata/normal_origin_info.txt',names=['time','idplace','mac'])
pid = 153
#place id
start_time = '2017-08-28 00:00:00'
end_time = '2017-09-28 00:00:00'
#自定  需要符合上述格式




start_date = '2017-08-28'
end_date = '2017-11-16'
weekdaylist =[]

sdate = datetime.datetime.strptime(start_date,'%Y-%m-%d')
edate = datetime.datetime.strptime(end_date,'%Y-%m-%d')
#字符串轉(zhuǎn)時(shí)間
while sdate<edate:
    weekdaylist.append(sdate.strftime('%Y-%m-%d'))
    sdate += datetime.timedelta(days=1)
#得到起止日期中的所有日期
wlist=[]
#目標(biāo)工作日
for i in xrange(len(weekdaylist)):
    if  datetime.datetime.strptime(weekdaylist[i],'%Y-%m-%d').weekday() == weekday-1:
        wlist.append(weekdaylist[i])
        i = i+7
#將符合weekday參數(shù)的工作日篩選出來(lái)
#其實(shí)上面weekday的代碼沒(méi)什么用锻弓，但蝌箍，代表了筆者的一段心路歷程w



def getInfoBypidtime(idplace,sdatetime,edatetime):

    records = pd.read_csv('./macdata/normalinfo_add.txt', names=['time','ts','year','month','day','hour','min','sec','weekday','min_hr','pid','mac','isholiday'],usecols=[0,10,11])
    #usecols選取指定列（這里選了三列，源數(shù)據(jù)時(shí)間列妓盲，pid列，mac列），若選取全部列則會(huì)出現(xiàn)memoryerror（說(shuō)多了都是淚）

    s = sdatetime[5:7]+sdatetime[8:10]
    e = edatetime[5:7]+edatetime[8:10]
    #輸出文件名規(guī)范
    idplace_time_info = records[ (records['pid'] == idplace) & (records['time'] > sdatetime) & (records['time'] < edatetime) ]
    #篩選符合條件的數(shù)據(jù)
    outputpath = './macdata/selectinfo/' + 'pid_' + str(idplace) + '_' + s + '-' + e + '.csv'
    idplace_time_info.to_csv(outputpath, header=False, index=False)


"""
規(guī)整前的數(shù)據(jù)：
2017-08-28 23:37:00,14,84742aa602e4
規(guī)整后的數(shù)據(jù)：
2017-08-28 23:37:00,ts,2017,08,28,23,37,00,1,2337,14,84742aa602e4,N
帶入的參數(shù)：
[1,2], [4,5],[[830,900],[1400,1600]]
"""
#以上是方法二的輸入?yún)?shù)規(guī)范

idplaces = [151,152,153]
weekday = [2,3,4]
#0是周一
timeranges = [[830,1200],[1400,1830]]

def getInfoBypid_wk_trange(idplaces,weekdays,timeranges):
    p = ''
    w = ''

    for i in xrange(len(idplaces)):
        p += str(idplaces[i]) + '_'

    for i in xrange(len(weekday)):
        w += '_' + str(weekday[i])
    #方法二輸出文件名規(guī)范

    outputpath2 = './macdata/selectinfo/' + 'pids_' + p + 'wkdays' + w + '.csv'

    rs = open(outputpath2, 'w')
    with open('./macdata/normalinfo_add.txt') as file:

       # new_line =[]

        for line in file:
            line = line.split(',')
            for i in xrange(len(timeranges)):
                if int(line[9])>timeranges[i][0] and int(line[9])<timeranges[i][1]:
                    if int(line[8]) in weekdays and int(line[10]) in idplaces:
                    #兩層篩選甚亭，第一層篩時(shí)間數(shù)組击胜，第二層篩pid
                        line[-2] = line[-2].strip('\n')
                        line[-1] = line[-1].strip('\n')
                        line2 = str(line)
                        line2 = line2.strip('[').strip(']').replace("'", "").replace(" ","")
                        #去空格偶摔，去單引號(hào)
                        line2 = line2[:10]+' '+line2[10:]
                        #時(shí)間重新拼接

                        rs.write(str(line2)+'\n')
    rs.close()

getInfoBypid_wk_trange(idplaces,weekday,timeranges)
# getInfoBypidtime(pid,start_time,end_time)

到這里位置，畢業(yè)設(shè)計(jì)的數(shù)據(jù)預(yù)處理工作已經(jīng)完成了
撒花辰斋！★,°:.☆(￣▽?zhuān)?/$:.°★ 。