from pyspark.sql import SparkSession
spark = SparkSession.builder.master("local[*]").appName("shuangyu").getOrCreate()
#讀取數(shù)據(jù)，并查看前5行
sc = spark.sparkContext
f = sc.textFile("/home/shuangyu/Downloads/ccFraud.csv")
f.take(5)
['"custID","gender","state","cardholder","balance","numTrans","numIntlTrans","creditLine","fraudRisk"',
'1,1,35,1,3000,4,14,2,0',
'2,2,2,1,0,9,0,18,0',
'3,2,2,1,0,27,9,16,0',
'4,1,15,1,0,12,0,5,0']
#準(zhǔn)備創(chuàng)建DataFrame
import pyspark.sql.types as typ
#獲取第一行，（第一行為列名）
header = f.first()
#數(shù)據(jù)去掉頭部扩然，并轉(zhuǎn)換成int
f = f.filter(lambda line:line != header).map(lambda line:[int(e) for e in line.split(",")])
#創(chuàng)建schema
cols = [*[typ.StructField(h[1:-1],typ.IntegerType(),True) for h in header.split(",")]]
schema = typ.StructType(cols)
#創(chuàng)建dataframe
df = spark.createDataFrame(f,schema)
df.cache()
df.printSchema()
root
|-- custID: integer (nullable = true)
|-- gender: integer (nullable = true)
|-- state: integer (nullable = true)
|-- cardholder: integer (nullable = true)
|-- balance: integer (nullable = true)
|-- numTrans: integer (nullable = true)
|-- numIntlTrans: integer (nullable = true)
|-- creditLine: integer (nullable = true)
|-- fraudRisk: integer (nullable = true)
#查看數(shù)據(jù)中性別比例
df.groupBy("gender").count().show()
+------+-------+
|gender| count|
+------+-------+
| 1|6178231|
| 2|3821769|
+------+-------+
#獲取指定字段的一些統(tǒng)計(jì)信息
df.describe("balance","numTrans","numIntlTrans").show()
+-------+-----------------+------------------+-----------------+
|summary| balance| numTrans| numIntlTrans|
+-------+-----------------+------------------+-----------------+
| count| 10000000| 10000000| 10000000|
| mean| 4109.9199193| 28.9351871| 4.0471899|
| stddev|3996.847309737258|26.553781024523122|8.602970115863904|
| min| 0| 0| 0|
| max| 41485| 100| 60|
+-------+-----------------+------------------+-----------------+
#相關(guān)性.
df.corr("balance","numTrans")
0.0004452314017265386
可視化
%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('ggplot')#樣式美化
# %matplotlib inline是用來(lái)使輸出圖像在notebook中輸出而不是單獨(dú)打開(kāi)一個(gè)窗口
#histogram 分桶，也就是畫(huà)直方圖中有多少個(gè)柱子
#返回的是一個(gè)tuple(List,List)袜漩，第一個(gè)list中是x，第二個(gè)是對(duì)應(yīng)的y
hist = df.select("balance").rdd.flatMap(lambda x: x).histogram(20)
print(type(hist))
hist
<class 'tuple'>
([0.0,
2074.25,
4148.5,
6222.75,
8297.0,
10371.25,
12445.5,
14519.75,
16594.0,
18668.25,
20742.5,
22816.75,
24891.0,
26965.25,
29039.5,
31113.75,
33188.0,
35262.25,
37336.5,
39410.75,
41485],
[3640021,
2258778,
1718633,
1035019,
601524,
343461,
192766,
104486,
54517,
27570,
13117,
5805,
2650,
1103,
358,
127,
45,
15,
1,
4])
#x里面有21個(gè)值湾碎，y有20個(gè)宙攻，所以x的最后一個(gè)要去掉（x軸最后是無(wú)窮區(qū)間）
data = {"bins":hist[0][:-1],"freq":hist[1]}
#width 柱子的x軸方向上的寬度，一般近似為x1-x0
plt.bar(data["bins"],data["freq"],width=2000)
plt.xlabel("X")
plt.ylabel("Y")
plt.title("Just For Test")
<matplotlib.text.Text at 0x7f48b3d6fc18>
#sampleBy根據(jù)指定字段分層抽樣
sample_data = df.sampleBy('gender',{1:0.0002,2:0.0002}).select("balance","numTrans","numIntlTrans")
sample_data.take(5)
[Row(balance=4000, numTrans=71, numIntlTrans=2),
Row(balance=0, numTrans=12, numIntlTrans=0),
Row(balance=3000, numTrans=44, numIntlTrans=0),
Row(balance=6000, numTrans=20, numIntlTrans=0),
Row(balance=3000, numTrans=3, numIntlTrans=13)]
#按字段將值分開(kāi)
data_multi ={elem:sample_data.select(elem).rdd.flatMap(lambda x:x).collect()\
for elem in ["balance","numTrans","numIntlTrans"]}
#畫(huà)散點(diǎn)圖胜茧，plt.scatter(x,y)
plt.scatter(data_multi["numTrans"],data_multi["balance"],c ='r',marker='o',label="numTrans-balance")
plt.scatter(data_multi["numIntlTrans"],data_multi["balance"],c ='b',marker="x",label="numInt-balance")
plt.grid(True)#網(wǎng)格
plt.xlabel("x value")
plt.ylabel("y value")
plt.legend(loc='upper right')#圖例的位置
plt.title("scatter view")
<matplotlib.text.Text at 0x7f48b3cdbf28>
