通過Spark Streaming毡咏，實時監(jiān)控HDFS目錄，發(fā)現(xiàn)有文件時，實時進行計算扰藕。
package com.spark.streaming
import org.apache.spark.SparkConf
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
/*
*
* 通過HDFS文件實時統(tǒng)計
*/
object HDFSWordCount extends App {
val conf = new SparkConf()
.setAppName("HDFSWordCount")
//.setMaster("hdfs://hadoop01:9000/")
val ssc = new StreamingContext(conf,Seconds(5))
val lines = ssc.textFileStream("hdfs://hadoop01:9000/wordcount_dir")
val words = lines.flatMap(line=>line.split(" "))
val paris = words.map((_,1))
val wordCount=paris.reduceByKey(_+_)
wordCount.print()
ssc.start()
ssc.awaitTermination()
ssc.stop()
}
