• 加載并查看數(shù)據(jù)基本情況

library(VIM)
data(sleep)
str(sleep)
summary(sleep)
head(sleep)

一陨囊、處理缺失值

• 查看NA的分布情況，有一個(gè)直觀了解

library('mice')
md.pattern(sleep)
matrixplot(sleep)

NA分布情況

• 根據(jù)NA的分布情況，獲取數(shù)據(jù)子集

#統(tǒng)計(jì)每一列NA的數(shù)量
na_flag <- apply(is.na(sleep), 2, sum)
# na_flag <- md.pattern(sleep,plot = F) %>% .[nrow(.),-ncol(.)]#同上
library('dplyr')
#獲取含有NA的列和不含NA的列
na_col = na_flag[na_flag > 0] %>% names()
full_col = setdiff(names(sleep),na_col)
# fill_col = names(sleep)[!(names(sleep) %in% na_col)]同上
# 獲取所有含有NA的行
na_df = sleep[!complete.cases(sleep),]
#獲取所有不含NA的行
full_df = na.omit(sleep)
#fill_df = sleep[complete.cases(sleep),]同上
#對(duì)變量進(jìn)行重新排序
sleep = sleep[,c(na_col,full_col)]

1. 刪除法

當(dāng)缺失值占比不大時(shí)严就，直接刪除缺失部分是最簡(jiǎn)單的辦法

1.1 針對(duì)特定變量刪除NA

sleep_del_Gest = sleep[!is.na(sleep$Gest),]
md.pattern(sleep_del_Gest)

對(duì)特定變量刪除NA后

1.2 刪除所有含NA的行

sleep_del_all = na.omit(sleep)
# sleep_del_all = sleep[complete.cases(sleep),]#同上
md.pattern(sleep_del_all)

刪除所有NA

1.3 觀察變化前后變量之間的相關(guān)性

library(PerformanceAnalytics)
chart.Correlation(sleep,histogram = T)
chart.Correlation(sleep_del_all,histogram = T)

刪除前變量相關(guān)性

刪除后變量相關(guān)性

2. 插補(bǔ)法

• 若缺失值占比過(guò)大荒辕，直接刪除會(huì)損失大量的信息汗销。這種情況應(yīng)該對(duì)缺失值進(jìn)行填補(bǔ)。
• 若屬性是連續(xù)的抵窒，則使用該屬性存在值的均值去插補(bǔ)缺失值弛针；
• 若屬性是離散的，則可取該屬性的眾數(shù)來(lái)插補(bǔ)缺失值李皇。

2.1. 使用均值,中值等針對(duì)某一個(gè)變量填值

#觀察數(shù)據(jù)分布情況
NonD_var = c(var = 'NonD',
             mean = mean(sleep$NonD,na.rm = T),
             median = median(sleep$NonD,na.rm = T),
             quantile(sleep$NonD,c(0,0.01,0.1,0.25,0.5,0.75,0.9,1),na.rm = T),
             max = max(sleep$NonD,na.rm = T),
             missing = sum(is.na(sleep$NonD)))
View(t(NonD_var))

變量NonD數(shù)據(jù)分布情況

#簡(jiǎn)單可視化
op <- par(mfrow = c(1,2))
hist(sleep$NonD,freq = F,col = 'lightblue',main = 'Befor')
#使用該變量現(xiàn)有數(shù)據(jù)的均值替換缺失值
library('Hmisc')
sleep$NonD = impute(sleep$NonD, fun = mean)#impute(x,2.5), impute(x,mean), impute(x,"random")
hist(NonD,freq = F,col = 'pink',main = 'After')

填值前后變量NonD數(shù)據(jù)分布變化

2.2. 基于kmeans均值算法填值

• 只適用于數(shù)值型缺失

####重新加載原始數(shù)據(jù)并對(duì)變量進(jìn)行排序
data("sleep",package = 'VIM')
sleep = sleep[,c(na_col,full_col)]

library('DMwR')
#以距離最近的3個(gè)值根據(jù)距離進(jìn)行加權(quán)平均來(lái)填值
sleep_fill_knn = knnImputation(sleep, k = 10, meth = 'weighAvg') 
md.pattern(sleep_fill_knn)

sleep_fill_knn = sleep_fill_knn[,names(sleep)]

#觀察變換前后數(shù)據(jù)的分布情況
op <- par(mfrow = c(5,2))
for(fct in na_col){
  hist(sleep[,fct],col = 'lightblue',freq = F,xlab = fct,main = 'Befor')
  hist(sleep_fill_knn[,fct],col = 'pink',freq = F,xlab = fct,main = 'After') 
}
par(op)

均值填值前后各變量數(shù)據(jù)分布情況

#觀察變換前后變量間的相關(guān)性
library(PerformanceAnalytics)
chart.Correlation(sleep,histogram = T)
chart.Correlation(sleep_fill_knn,histogram = T)

before_knnImputation

after_knnImputation

2.3. 基于回歸算法填值

f = as.formula(paste(paste(na_col,collapse = ' + '),'~',paste(fill_col,collapse = ' + ')))
sleep_fill_reg = regressionImp(f,data = sleep)
#這里的formular = y ~ .,y是response variable削茁，即要插補(bǔ)的變量。
#我這里使用了所有包含NA的變量來(lái)對(duì)所有不含NA的變量進(jìn)行回歸掉房，有些極端茧跋，僅供參考。

sleep_fill_reg = sleep_fill_reg[,names(sleep)]
md.pattern(sleep_fill_reg)

#觀察變換前后數(shù)據(jù)的分布情況卓囚，變量間的相關(guān)性
library(PerformanceAnalytics)
chart.Correlation(sleep,histogram = T)
chart.Correlation(sleep_fill_reg,histogram = T)

befor_regressionImp

after_regressionImp

2.4. 基于隨機(jī)森林替換

if(!require('randomForest'))(
  install.packages('randomForest')
)
sleep_fill_rf = rfImpute(Danger ~ .,sleep) 
sleep_fill_rf = sleep_fill_rf[,names(sleep)]
#觀察變換前后數(shù)據(jù)的分布情況厌衔，變量間的相關(guān)性
library(PerformanceAnalytics)
chart.Correlation(sleep,histogram = T)
chart.Correlation(sleep_fill_rf,histogram = T)

befor_rf

after_rf

二、處理異常值

1.1 單變量異常值檢測(cè)

lb = c('ggplot2','reshape2','dplyr')
lapply(lb,require,character.only = T)

boxplot(sleep_fill_rf,frame = T)

sleep_fill_rf %>% melt() %>%
  ggplot(aes(NULL,value)) +
  geom_boxplot(aes(fill = variable)) +
  facet_wrap(variable ~. , scales = 'free_y')

sigular_boxplot

sigular_ggboxplot

1.2 蓋帽法處理異常值

#采用蓋帽法捍岳，用10%處的數(shù)據(jù)覆蓋分布在10%以下的數(shù)據(jù)富寿，用90%處的數(shù)據(jù)覆蓋分布在99%以上的數(shù)據(jù)。
#這里的10%和90%取值有些極端锣夹，及供參考页徐。
block<-function(x,lower=T,upper=T){
  if(lower){
    q1<-quantile(x,0.1)
    x[x<=q1]<-q1
  }
  if(upper){
    q99<-quantile(x,0.90)
    x[x>q99]<-q99
  }
  return(x)
}

sleep_fill_rf_blk = sapply(sleep_fill_rf,block)
boxplot(sleep_fill_rf_blk,frame = T)

after_block

• 參考
  apricoter