數(shù)據(jù)導(dǎo)入與查看

# -*- coding: utf-8 -*-
# %%time
# from pyhive import presto
import pandas as pd
import numpy as np
import warnings
import os
data=pd.read_csv('*/全域風(fēng)險(xiǎn).csv')
data.head(2)
# label= pd.DataFrame(list(result),columns=columns_names)
# label.to_csv('/data/ljk/baixin.csv',index=False)

數(shù)據(jù)篩選

data2=data[data['fina_date']<'2020-01-01']
feature=['num_id','zhiye','weiyue','gongzhai','qingchang','zhuxing','lvyue','shouxin','xiaofei','xingqu','chengzhang']
data2=data2[feature]
data2.head()

scorecardpy Python包的使用

import scorecardpy as sc
import matplotlib.pyplot as plt
%matplotlib inline

plt.show()

bins_new=sc.woebin(data_new.loc[data_new.overdue!=-1,['zhiye','overdue']], y="overdue")
woebin_plot=sc.woebin_plot(bins_new)

woebin_plot

結(jié)果編輯

# data_new = data_new.drop(['flag','var_name'],axis=1)

merge_result_total = pd.DataFrame()
for cl in data_new.columns[1:]:  
    x=data_new[data_new[cl]>=0][cl]
    if len(set(x))>=10:
        value_bins=pd.qcut(x,5,duplicates='drop',retbins=True)[0]
        data_new['flag']=value_bins #攔截點(diǎn)打標(biāo)
        data_new['var_name']= cl  #變量 
        tmp = data_new[['var_name','flag','overdue']]
        tmp.rename(columns={'overdue':'label'},inplace=True)
        result_stp=tmp.groupby(['var_name','flag']).count() #攔截?cái)?shù) 
        result_pos=tmp.groupby(['var_name','flag'])['label'].sum().to_frame()  #黑樣本
        result_neg=tmp[tmp['label']==0].groupby(['var_name','flag'])['label'].count().to_frame() #白樣本
        merge_result=result_stp.merge(result_pos,how='left',on=['var_name','flag']).merge(result_neg,how='left',on=['var_name','flag'])
        merge_result.reset_index(inplace=True)
        merge_result.rename(columns={'label_x':'stp','label_y':'pos','label':'neg'},inplace=True) 
        
        merge_result['rank']=pd.Series([i+1 for i in range(merge_result.shape[0])])
        merge_result.sort_values(by=['rank'],axis=0,ascending=True,inplace=True,na_position='last')
        
        merge_result['cunsum_pos']=merge_result.groupby('var_name')['pos'].cumsum()
        merge_result['cunsum_neg']=merge_result.groupby('var_name')['neg'].cumsum()
        merge_result['cunsum_stp']=merge_result.groupby('var_name')['stp'].cumsum()
        
        merge_result['total_pos']=merge_result[merge_result['rank']== merge_result.shape[0]]['cunsum_pos'].values[0]
        merge_result['total_neg']=merge_result[merge_result['rank']== merge_result.shape[0]]['cunsum_neg'].values[0]
        merge_result['total_stp']=merge_result[merge_result['rank']== merge_result.shape[0]]['cunsum_stp'].values[0]
    res = merge_result

    res['intercept']=res['stp']/res['total_stp'] # 區(qū)間攔截率
    res['precision']=res['pos']/res['stp'] # 準(zhǔn)確率
    res['recall']=res['pos']/res['total_pos']  #召回率
    res['Disturb']=res['neg']/res['total_neg'] #打擾率
    res['cum_precision']=res['cunsum_pos']/res['cunsum_stp']  # 累計(jì)準(zhǔn)確率
    res['avg_precision']=res['total_pos']/res['total_stp']
    res['cum_recall']=res['cunsum_pos']/res['total_pos'] # 累計(jì)召回率
    res['cum_Disturb']=res['cunsum_neg']/res['total_neg'] # 累計(jì)打擾率

    res['ks']=res['cum_recall']-res['cum_Disturb']
    res['ks_max']=res.groupby('var_name')['ks'].max().values[0]

    rs=res.drop_duplicates(subset=None, keep='first', inplace=False) #去重
    
    merge_result_total = merge_result_total.append(rs)
    
merge_result_total.rename(columns={'var_name':'變量','flag':'攔截區(qū)間','stp':'攔截樣本數(shù)','pos':'黑樣本數(shù)','neg':'白樣本數(shù)','cunsum_pos':'累計(jì)黑樣本數(shù)','cunsum_neg':'累計(jì)白樣本數(shù)','cunsum_stp':'累計(jì)攔截?cái)?shù)','intercept':'攔截率','precision':'準(zhǔn)確率','recall':'召回率','Disturb':'打擾率','cum_precision':'累計(jì)準(zhǔn)確率','avg_precision':'平均準(zhǔn)確率','cum_recall':'累計(jì)召回率','cum_Disturb':'累計(jì)打擾率','ks':'ks區(qū)間值','ks_max':'ks值','total_pos':'總黑樣本','total_neg':'總白樣本','total_stp':'總樣本'},inplace=True)
merge_result_total.to_csv('*/quanyumob3_result0421.csv',header=True,index=False)

merge_result_total

ks曲線函數(shù)

調(diào)用方法

ks=PlotKS(data_new3['zhiye'],data_new3['overdue'],n=20,asc=True)
ks
plt.show

import pandas as pd
import matplotlib.pyplot as plt
####################### PlotKS ##########################
def PlotKS(preds, labels, n=20, asc=True):
    
    # preds is score: asc=1
    # preds is prob: asc=0
    
    pred = preds  # 預(yù)測(cè)值
    bad = labels  # 取1為bad, 0為good
    ksds = pd.DataFrame({'bad': bad, 'pred': pred})
    ksds['good'] = 1 - ksds.bad
    
    if asc == 1:
        ksds1 = ksds.sort_values(by=['pred', 'bad'], ascending=[True, True])
    elif asc == 0:
        ksds1 = ksds.sort_values(by=['pred', 'bad'], ascending=[False, True])
    ksds1.index = range(len(ksds1.pred))
    ksds1['cumsum_good1'] = 1.0*ksds1.good.cumsum()/sum(ksds1.good)
    ksds1['cumsum_bad1'] = 1.0*ksds1.bad.cumsum()/sum(ksds1.bad)
    
    if asc == 1:
        ksds2 = ksds.sort_values(by=['pred', 'bad'], ascending=[True, False])
    elif asc == 0:
        ksds2 = ksds.sort_values(by=['pred', 'bad'], ascending=[False, False])
    ksds2.index = range(len(ksds2.pred))
    ksds2['cumsum_good2'] = 1.0*ksds2.good.cumsum()/sum(ksds2.good)
    ksds2['cumsum_bad2'] = 1.0*ksds2.bad.cumsum()/sum(ksds2.bad)
    
    # ksds1 ksds2 -> average
    ksds = ksds1[['cumsum_good1', 'cumsum_bad1']]
    ksds['cumsum_good2'] = ksds2['cumsum_good2']
    ksds['cumsum_bad2'] = ksds2['cumsum_bad2']
    ksds['cumsum_good'] = (ksds['cumsum_good1'] + ksds['cumsum_good2'])/2
    ksds['cumsum_bad'] = (ksds['cumsum_bad1'] + ksds['cumsum_bad2'])/2
    
    # ks
    ksds['ks'] = ksds['cumsum_bad'] - ksds['cumsum_good']
    ksds['tile0'] = range(1, len(ksds.ks) + 1)
    ksds['tile'] = 1.0*ksds['tile0']/len(ksds['tile0'])
    
    qe = list(np.arange(0, 1, 1.0/n))
    qe.append(1)
    qe = qe[1:]
    
    ks_index = pd.Series(ksds.index)
    ks_index = ks_index.quantile(q = qe)
    ks_index = np.ceil(ks_index).astype(int)
    ks_index = list(ks_index)
    
    ksds = ksds.loc[ks_index]
    ksds = ksds[['tile', 'cumsum_good', 'cumsum_bad', 'ks']]
    ksds0 = np.array([[0, 0, 0, 0]])
    ksds = np.concatenate([ksds0, ksds], axis=0)
    ksds = pd.DataFrame(ksds, columns=['tile', 'cumsum_good', 'cumsum_bad', 'ks'])
    
    ks_value = ksds.ks.max()
    ks_pop = ksds.tile[ksds.ks.idxmax()]
    print ('ks_value is ' + str(np.round(ks_value, 4)) + ' at pop = ' + str(np.round(ks_pop, 4)))
    
    # chart
    plt.plot(ksds.tile, ksds.cumsum_good, label='cum_good',
                         color='blue', linestyle='-', linewidth=2)
                         
    plt.plot(ksds.tile, ksds.cumsum_bad, label='cum_bad',
                        color='red', linestyle='-', linewidth=2)
                        
    plt.plot(ksds.tile, ksds.ks, label='ks',
                   color='green', linestyle='-', linewidth=2)
                       
    plt.axvline(ks_pop, color='gray', linestyle='--')
    plt.axhline(ks_value, color='green', linestyle='--')
    plt.axhline(ksds.loc[ksds.ks.idxmax(), 'cumsum_good'], color='blue', linestyle='--')
    plt.axhline(ksds.loc[ksds.ks.idxmax(),'cumsum_bad'], color='red', linestyle='--')
    plt.title('KS=%s ' %np.round(ks_value, 4) +  
                'at Pop=%s' %np.round(ks_pop, 4), fontsize=15)
    return ksds