1枢劝、監(jiān)控ip井联、port

###################【配置文件】####################
# 定義這個agent中各組件的名字
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# 描述和配置source組件：r1
a1.sources.r1.type = netcat
a1.sources.r1.bind = 192.168.52.130
a1.sources.r1.port = 44444
# 描述和配置sink組件：k1
a1.sinks.k1.type = logger
# 描述和配置channel組件，此處使用是內(nèi)存緩存的方式
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# 描述和配置source  channel   sink之間的連接關(guān)系
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
########################【啟動腳本】###############
bin/flume-ng agent \
    -c conf \
    -f conf/netcat-logger.conf \
    -n a1  \
    -Dflume.root.logger=INFO,console
-c conf   指定flume自身的配置文件所在目錄
-f conf/netcat-logger.con  指定我們所描述的采集方案
-n a1  指定我們這個agent的名字

2您旁、監(jiān)控文件目錄

#######################【配置文件】###################
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
##注意：不能往監(jiān)控目中重復(fù)丟同名文件
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /export/servers/dirfile
a1.sources.r1.fileHeader = true

# Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.channel = c1
a1.sinks.k1.hdfs.path = hdfs://node01:8020/spooldir/files/%y-%m-%d/%H%M/
a1.sinks.k1.hdfs.filePrefix = events-
a1.sinks.k1.hdfs.round = true
a1.sinks.k1.hdfs.roundValue = 10
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.rollInterval = 3
a1.sinks.k1.hdfs.rollSize = 20
a1.sinks.k1.hdfs.rollCount = 5
a1.sinks.k1.hdfs.batchSize = 1
a1.sinks.k1.hdfs.useLocalTimeStamp = true

#生成的文件類型烙常，默認(rèn)是Sequencefile，可用DataStream，則為普通文本
a1.sinks.k1.hdfs.fileType = DataStream
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

Channel參數(shù)解釋：
capacity：默認(rèn)該通道中最大的可以存儲的event數(shù)量
trasactionCapacity：每次最大可以從source中拿到或者送到sink中的event數(shù)量
keep-alive：event添加到通道中或者移出的允許時間

########################【啟動腳本】#############
bin/flume-ng agent \
    -c ./conf \
    -f ./conf/spooldir.conf \
    -n a1 \
    -Dflume.root.logger=INFO,console

3蚕脏、采集文件到HDFS

###################【配置文件】################
agent1.sources = source1
agent1.sinks = sink1
agent1.channels = channel1
 
# Describe/configure tail -F source1
agent1.sources.source1.type = exec
agent1.sources.source1.command = tail -F /export/servers/taillogs/access_log
agent1.sources.source1.channels = channel1

# Describe sink1
agent1.sinks.sink1.type = hdfs
#a1.sinks.k1.channel = c1
agent1.sinks.sink1.hdfs.path = hdfs://node01:8020/weblog/flume-collection/%y-%m-%d/%H-%M
agent1.sinks.sink1.hdfs.filePrefix = access_log
agent1.sinks.sink1.hdfs.maxOpenFiles = 5000
agent1.sinks.sink1.hdfs.batchSize= 100
agent1.sinks.sink1.hdfs.fileType = DataStream
agent1.sinks.sink1.hdfs.writeFormat =Text
agent1.sinks.sink1.hdfs.rollSize = 102400
agent1.sinks.sink1.hdfs.rollCount = 1000000
agent1.sinks.sink1.hdfs.rollInterval = 60
agent1.sinks.sink1.hdfs.round = true
agent1.sinks.sink1.hdfs.roundValue = 10
agent1.sinks.sink1.hdfs.roundUnit = minute
agent1.sinks.sink1.hdfs.useLocalTimeStamp = true
# Use a channel which buffers events in memory
agent1.channels.channel1.type = memory
agent1.channels.channel1.keep-alive = 120
agent1.channels.channel1.capacity = 500000
agent1.channels.channel1.transactionCapacity = 600
 
# Bind the source and sink to the channel
agent1.sources.source1.channels = channel1
agent1.sinks.sink1.channel = channel1

##############【啟動腳本】####################
bin/flume-ng agent \
    -c conf \
    -f conf/tail-file.conf \
    -n agent1  \
    -Dflume.root.logger=INFO,console