//得到了一個(gè)rdd
val textFile = sc.textFile("file:///xuzhang/home/Cloud/spark/README.md");
//得到其中的rdd總數(shù)
textFile.count()
//得到第一條數(shù)據(jù)
textFile.first()
//filter 過(guò)濾數(shù)據(jù)演熟，spark中叫做rdd轉(zhuǎn)換
val lineWithSpark = textFile.filter(line=>line.contains("Spark"));
//或者
val lineWithSpark = textFile.filter(_.contains("Spark");
//map reduce
textFile.map(_.split(" ").size).reduce((a,b)=>if (a > b) a else b))
//調(diào)用java的方法統(tǒng)計(jì)
import java.lang.Math
textFile.map(_.split(" ").size).reduce((a,b)=>Math.max(a,b))
//spark 的wordCount
textFile.flatMap(_.split(" ")).map(word=>(word,1)).reduceByKey((a,b)=>a+b)
編程之scala版
/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
object SimpleApp {
def main(args: Array[String]) {
val logFile = "YOUR_SPARK_HOME/README.md"
// Should be some file on your system
val conf = new SparkConf().setAppName("Simple Application")
val sc = new SparkContext(conf)
val logData = sc.textFile(logFile, 2).cache()
val numAs = logData.filter(line => line.contains("a")).count()
val numBs = logData.filter(line => line.contains("b")).count()
println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
}
}
編程之python版
"""SimpleApp.py"""
from pyspark import SparkContext
logFile = "YOUR_SPARK_HOME/README.md" # Should be some file on your system
sc = SparkContext("local", "Simple App")
logData = sc.textFile(logFile).cache()
numAs = logData.filter(lambda s: 'a' in s).count()
numBs = logData.filter(lambda s: 'b' in s).count()
print("Lines with a: %i, lines with b: %i" % (numAs, numBs))
