1. 介紹
- 將Flume客戶端和真正任務(wù)配置的文件夾隔離開
- 通過啟動命令指定每個任務(wù)的執(zhí)行日志
- 真正任務(wù)配置中的可變參數(shù) 均采用傳參使用妨退，用于生產(chǎn)開發(fā)測試不同環(huán)境的不同參數(shù)
2. 例子
- 配置
flume_job_site_conf.properties
# Kafka 連接地址
kafkaCluster=192.168.xx.x1:9092,192.168.xx.x2:9092,192.168.xx.x3:9092
kafkaCluster_acl=192.168.xx.x1:9092,192.168.xx.x2:9092,192.168.xx.x3:9092
# Kafka SASL加密的賬號密碼
kfk_user=flume
kfk_pwd=123456
# Flume 服務(wù)端IP
flumeServerIP1=192.168.xx.x1
flumeServerIP2=192.168.xx.x1
# HDFS 集群名稱
hadoopClusterName=mycluster
hdfsFileSuffix=1
- 父腳本
flume_common.sh
#!/bin/bash
source /etc/profile
usage="Usage: ./***.sh ( start | stop | status )"
# if no args specified, show usage
if [ $# -eq 0 ]; then
echo $usage
exit 1
fi
# Agent 名稱
agentName=$2
# Flume 任務(wù)配置路徑
jobLocation=$3
# 任務(wù)日志以及Channel路徑
logDir=$4
# 監(jiān)控端口
monitoringPort=$5
# Flume 客戶端路徑
flumeClient="${FLUME_HOME}"
# Flume任務(wù)相關(guān)的連接參數(shù)
file="${FLUME_JOB_OEM_CONFIG_PATH}/resources_site/flume_job_site_conf.properties"
if [ ! -f "$file" ]; then
echo "$file not found."
exit 1
fi
while IFS='=' read -r key value
do
key=$(echo $key | tr '.' '_')
eval ${key}=\${value}
done < "$file"
# Kafka 連接地址
kafkaCluster="${kafkaCluster}"
kafkaCluster_acl="${kafkaCluster_acl}"
kfk_user="${kfk_user}"
kfk_pwd="${kfk_pwd}"
hadoopClusterName="${hadoopClusterName}"
hdfsFileSuffix="${hdfsFileSuffix}"
bdCollectPath="${bdCollectPath}"
bdOEMCollectPath="${bdOEMCollectPath}"
# 參數(shù)說明：
# --conf/-c：表示配置文件存儲在 conf/目錄
# --name/-n：表示給 agent 起名為 a1
# --conf-file/-f：表示任務(wù)文件的位置
# -D ：表示 flume 運行時動態(tài)修改配置文件
# -Dflume.root.logger=INFO,console ：動態(tài)修改log4j.properties中設(shè)置了日志打印級別以及位置
# -Dflume.log.dir ：動態(tài)修改log4j.properties中設(shè)置了日志文件存放位置
# {
# propertiesImplementation=org.apache.flume.node.EnvVarResolverProperties 固定的，
# 是為了將 exec_log_path=$logDir 傳進(jìn) jobName 文件中使用
# exec_log_path=$logDir 這個必須寫在前面
# }
case $1 in
"start"){
echo "================= Agent名為 ${agentName} 的Flume 開始采集 ==============="
exec_log_path=${logDir} bdCollectPath=${bdCollectPath} bdOEMCollectPath=${bdOEMCollectPath} kafkaCluster=${kafkaCluster} kafkaCluster_acl=${kafkaCluster_acl} kfk_user=${kfk_user} kfk_pwd=${kfk_pwd} hadoopClusterName=${hadoopClusterName} hdfsFileSuffix=${hdfsFileSuffix} nohup ${flumeClient}/bin/flume-ng agent -n ${agentName} -c ${flumeClient}/conf/ -f ${jobLocation} -Dflume.log.dir=${logDir} -Dflume.monitoring.type=http -Dflume.monitoring.port=${monitoringPort} -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties > ${logDir}/run.log 2>&1 &
};;
"stop"){
echo "================= Agent名為 ${agentName} 的Flume 停止采集 ==============="
ps -ef | grep ${jobLocation} | grep -v grep | grep -v "flume_common.sh" | awk '{print $2}' | xargs kill -15
};;
"status"){
echo "================= 查詢 Agent名為 ${agentName} 的Flume 進(jìn)程 ==============="
echo `ps -ef | grep ${jobLocation} | grep -v grep | grep -v "flume_common.sh"`
};;
*){
echo ${usage}
};;
esac
- 啟動任務(wù)的腳本
flume_app_info_to_hdfs.sh
#!/bin/bash
source /etc/profile
usage="Usage: ./***.sh ( start | stop | status )"
# if no args specified, show usage
if [ $# -eq 0 ]; then
echo $usage
exit 1
fi
# 監(jiān)控端口
monitoringPort=34551
# Agent 名稱
agentName="a_app_info_to_hdfs"
# Flume 任務(wù)配置路徑
jobLocation="${FLUME_JOB_OEM_CONFIG_PATH}/kafka_to_hdfs/flume_app_info_to_hdfs/flume_app_info_to_hdfs.conf"
# 任務(wù)日志以及Channel路徑
logDir="${FLUME_JOB_OEM_CONFIG_PATH}/log_channel_datas/${agentName}"
sh ${FLUME_JOB_OEM_CONFIG_PATH}/resources_site/flume_common.sh $1 ${agentName} ${jobLocation} ${logDir} ${monitoringPort}
