1圈澈、導(dǎo)入數(shù)據(jù) 略

2、查看數(shù)據(jù)

train_data.columns
test_data.columns
# isDefault是Y變量，test_data多了n2.2和n2.3字段

train_data.info()

train_data.describe()

2.1查看缺失值的兩種方法

missing = train_data.isnull().sum()/len(train_data)
missing = missing[missing > 0]
missing.sort_values(inplace = True)
missing.plot.bar()

缺失值可視化

def missing_data(data):
    total = data.isnull().sum().sort_values(ascending=False)
    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending=False)
    return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data(train_data).head(20)

缺失值表格

2.2查看只有一個(gè)值的特征

①nunique() 返回唯一值個(gè)數(shù)

②unique() 返回唯一值array

one_value_fea = [col for col in train_data.columns if train_data[col].nunique()<=1]
one_value_fea_test = [col for col in test_data.columns if test_data[col].nunique()<=1]

print(one_value_fea)
print(one_value_fea_test)
#['policyCode']

print(f'There are {len(one_value_fea)} columns in train dataset with one unique value.')
print(f'There are {len(one_value_fea_test)} columns in test dataset with one unique value.')
#There are 1 columns in test dataset with one unique value.

2.3根據(jù)數(shù)據(jù)類型進(jìn)行特征分類

分成category叶摄，numerical兩類，其中numerical再細(xì)分成連續(xù)和離散兩類

numerical_fea = list(train_data.select_dtypes(exclude=['object']).columns)
catagory_fea = list(filter(lambda x:x not in numeric_fea, list(train_data.columns)))

將數(shù)值型變量分類為連續(xù)性變量和離散型變量
#過濾數(shù)值型類別特征
def get_numerical_serial_fea(data,feas):
    numerical_serial_fea = []
    numerical_noserial_fea = []
    for fea in feas:
        temp = data[fea].nunique()
        if temp <= 20:
            numerical_noserial_fea.append(fea)
            continue
        numerical_serial_fea.append(fea)
    return numerical_serial_fea,numerical_noserial_fea
numerical_serial_fea,numerical_noserial_fea = get_numerical_serial_fea(train_data,numerical_fea)

#可以和后面的單因素分析結(jié)合
離散型數(shù)值數(shù)據(jù)的閾值可以取10,20,30安拟，根據(jù)需要來
numerical_noserial_fea
#['term', 'homeOwnership', 'verificationStatus', 'isDefault', 'purpose', 'pubRecBankruptcies', 'initialListStatus', 'applicationType', 'policyCode', 'n11', 'n12']

2.4連續(xù)型變量分布

原代碼運(yùn)行時(shí)間比較長蛤吓，且沒有去掉id編碼，所以也嘗試了另一種代碼糠赦。比較粗糙会傲，但是也符合查看各變量分布的需求。

f = pd.melt(train_data, value_vars=numerical_serial_fea)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False)
g = g.map(sns.distplot, "value")

train_data[numerical_serial_fea].drop('id',axis=1).hist(bins=30, figsize=(15,20))
plt.show()

各變量分布

2.5樣本均衡性

#1表示違約拙泽，0表示正常還款
temp = train_data['isDefault'].value_counts()
df = pd.DataFrame({'labels':temp.index, 
                   'values':temp.values})
plt.figure(figsize = (6,6))
plt.title('違約情況分布')
sns.set_color_codes(palette='bright')
sns.barplot(x = 'labels', y = 'values', data=df)
plt.show()

#總體違約率
train_data['isDefault'].mean()
#0.1995125

總體違約率19.95%淌山，屬于不均衡樣本。

2.6查看單因素違約率

def plot_stats(feature,label_rotation=False,horizontal_layout=True):
    temp = train_data[feature].value_counts()
    df1 = pd.DataFrame({feature: temp.index,'Number of contracts': temp.values})

    # 違約率[0,1]顾瞻，可以用平均值計(jì)算比例
    cat_perc = train_data[[feature, 'isDefault']].groupby([feature],as_index=False).mean()
    cat_perc.sort_values(by='isDefault', ascending=False, inplace=True)
    
    #ncols泼疑，2列1行，nrows荷荤，1列2行
    if(horizontal_layout):
        fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(12,6))
    else:
        fig, (ax1, ax2) = plt.subplots(nrows=2, figsize=(12,14))
    sns.set_color_codes("pastel")
    s = sns.barplot(ax=ax1, x = feature, y="Number of contracts",data=df1)
    if(label_rotation):
        s.set_xticklabels(s.get_xticklabels(),rotation=90)
    
    s = sns.barplot(ax=ax2, x = feature, y='isDefault', order=cat_perc[feature], data=cat_perc)
    if(label_rotation):
        s.set_xticklabels(s.get_xticklabels(),rotation=90)
    #增加平均逾期率參考線
    plt.axhline(0.1995125, color = 'red', linewidth = 2, linestyle='--')
    plt.ylabel('Percent of target with value 1 [%]', fontsize=10)
    plt.tick_params(axis='both', which='major', labelsize=10)

    plt.show()

plot_stats('grade')
plot_stats('term')
plot_stats('homeOwnership')
plot_stats('employmentLength',1,0)

'grade'

'term'

'homeOwnership'

'employmentLength'

2.7數(shù)值型變量不同y值分布差異

def plot_distribution_comp(var,nrow=2):
    
    i = 0
    t1 = train_data.loc[train_data['isDefault'] != 0]
    t0 = train_data.loc[train_data['isDefault'] == 0]

    sns.set_style('whitegrid')
    plt.figure()
    fig, ax = plt.subplots(nrow,2,figsize=(12,6*nrow))

    for feature in var:
        i += 1
        plt.subplot(nrow,2,i)
        sns.kdeplot(t1[feature].dropna(), bw=0.1, label="isDefault = 1")
        sns.kdeplot(t0[feature].dropna(), bw=0.1, label="isDefault = 0")
        plt.ylabel('Density plot', fontsize=12)
        plt.xlabel(feature, fontsize=12)
        locs, labels = plt.xticks()
        plt.tick_params(axis='both', which='major', labelsize=12)
    plt.show()

var = ['purpose', 'homeOwnership', 'dti', 'revolUtil', 'delinquency_2years', 'regionCode']
plot_distribution_comp(var, nrow=3)

9.png

選取的幾個(gè)特征違約與否的分布無明顯差異蕴纳，需要后續(xù)特征處理再進(jìn)行區(qū)分会油。

2.8類別型變量不同y值分布差異

train_data1 = train_data.loc[train_data['isDefault'] != 0]
train_data0 = train_data.loc[train_data['isDefault'] == 0]

fig,((ax1,ax2),(ax3,ax4)) = plt.subplots(2,2,figsize=(12,8))
train_data1.groupby('grade')['grade'].count().plot(kind='barh', ax=ax1, 
                                                   title='Fraud-grade distribution')
train_data0.groupby('grade')['grade'].count().plot(kind='barh', ax=ax2, 
                                                   title='non_Fraud-grade distribution')
train_data1.groupby('employmentLength')['employmentLength'].count().plot(kind='barh', ax=ax3, 
                                                   title='Fraud-employmentLength distribution')
train_data0.groupby('employmentLength')['employmentLength'].count().plot(kind='barh', ax=ax4, 
                                                   title='non_Fraud-employmentLength distribution')

10.png

grade有比較明顯的差異，和單因素違約率的結(jié)果也相符

2.9比較train_data和test_data時(shí)間段是否一致

#轉(zhuǎn)化成時(shí)間格式  issueDateDT特征表示數(shù)據(jù)日期離數(shù)據(jù)集中日期最早的日期（2007-06-01）的天數(shù)
train_data['issueDate'] = pd.to_datetime(train_data['issueDate'],format='%Y-%m-%d')
startdate = datetime.datetime.strptime('2007-06-01', '%Y-%m-%d')
train_data['issueDateDT'] = train_data['issueDate'].apply(lambda x: x-startdate).dt.days
#轉(zhuǎn)化成時(shí)間格式
test_data['issueDate'] = pd.to_datetime(test_data['issueDate'],format='%Y-%m-%d')
startdate = datetime.datetime.strptime('2007-06-01', '%Y-%m-%d')
test_data['issueDateDT'] = test_data['issueDate'].apply(lambda x: x-startdate).dt.days
plt.hist(train_data['issueDateDT'], label='train')
plt.hist(test_data['issueDateDT'], label='test')
plt.legend()
plt.title('Distribution of issueDateDT dates')

11.png

時(shí)間分布一致古毛，后面可以直接建模擬合翻翩。

通過EDA部分，可以從宏觀角度對(duì)每組數(shù)據(jù)有一個(gè)直觀的認(rèn)識(shí)稻薇，通過與業(yè)務(wù)的討論体斩，確定合適的特征變量、數(shù)據(jù)現(xiàn)象颖低，可以讓EDA部分更有實(shí)際意義，簡(jiǎn)化數(shù)據(jù)清理過程弧烤。