Elastic Stack 5.1 部署(2)

從其他服務(wù)器抓取日志存到ELK
業(yè)務(wù)服務(wù)器上配置logstash

[root@tzgsqvapp01 app]# tar zxvf logstash-2.4.0.tar.gz
[root@tzgsqvapp01 app]# rm -rf logstash-2.4.0.tar.gz
[root@tzgsqvapp01 app]# ln -s logstash-2.4.0 logstash
[root@tzgsqvapp01 app]# useradd logstash -M -s /sbin/nologin
[root@tzgsqvapp01 app]# mkdir /opt/logs/logstash/
[root@tzgsqvapp01 app]# chown -R logstash:logstash /opt/logs/logstash/
[root@tzgsqvapp01 app]# chown -R logstash:logstash /opt/app/logstash-2.4.0/
[root@tzgsqvapp01 app]# mkdir /opt/app/logstash/conf.d
[root@tzgsqvapp01 app]# chown -R logstash.logstash opt/app/logstash/conf.d
[root@tzgsqvapp01 app]# vi /etc/init.d/logstash

#!/bin/sh
# Init script for logstash
# Maintained by Elasticsearch
# Generated by pleaserun.
# Implemented based on LSB Core 3.1:
# * Sections: 20.2, 20.3
#
### BEGIN INIT INFO
# Provides: logstash
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description:
# Description: Starts Logstash as a daemon.
### END INIT INFO

PATH=/sbin:/usr/sbin:/bin:/usr/bin
export PATH

if [ `id -u` -ne 0 ]; then
echo "You need root privileges to run this script"
exit 1
fi

name=logstash
pidfile="/var/run/$name.pid"

LS_USER=logstash
LS_GROUP=logstash
LS_HOME=/opt/app/logstash
LS_HEAP_SIZE="500m"
LS_LOG_DIR=/opt/logs/logstash
LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
LS_CONF_DIR=/opt/app/logstash/conf.d
LS_OPEN_FILES=16384
LS_NICE=19
LS_OPTS=""


[ -r /etc/default/$name ] && . /etc/default/$name
[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name

program=/opt/app/logstash/bin/logstash
args="agent -f ${LS_CONF_DIR} -l ${LS_LOG_FILE} ${LS_OPTS}"

start() {

LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}"
HOME=${LS_HOME}
export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING

# chown doesn't grab the suplimental groups when setting the user:group - so we have to do it for it.
# Boy, I hope we're root here.
SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed 's/,$//'; echo '')

if [ ! -z $SGROUPS ]
then
EXTRA_GROUPS="--groups $SGROUPS"
fi

# set ulimit as (root, presumably) first, before we drop privileges
ulimit -n ${LS_OPEN_FILES}

# Run the program!
nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c "
cd $LS_HOME
ulimit -n ${LS_OPEN_FILES}
exec \"$program\" $args
" > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

# Generate the pidfile from here. If we instead made the forked process
# generate it there will be a race condition between the pidfile writing
# and a process possibly asking for status.
echo $! > $pidfile

echo "$name started."
return 0
}

stop() {
# Try a few times to kill TERM the program
if status ; then
pid=`cat "$pidfile"`
echo "Killing $name (pid $pid) with SIGTERM"
kill -TERM $pid
# Wait for it to exit.
for i in 1 2 3 4 5 ; do
echo "Waiting $name (pid $pid) to die..."
status || break
sleep 1
done
if status ; then
if [ "$KILL_ON_STOP_TIMEOUT" -eq 1 ] ; then
echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss."
kill -KILL $pid
echo "$name killed with SIGKILL."
else
echo "$name stop failed; still running."
fi
else
echo "$name stopped."
fi
fi
}

status() {
if [ -f "$pidfile" ] ; then
pid=`cat "$pidfile"`
if kill -0 $pid > /dev/null 2> /dev/null ; then
# process by this pid is running.
# It may not be our pid, but that's what you get with just pidfiles.
# TODO(sissel): Check if this process seems to be the same as the one we
# expect. It'd be nice to use flock here, but flock uses fork, not exec,
# so it makes it quite awkward to use in this case.
return 0
else
return 2 # program is dead but pid file exists
fi
else
return 3 # program is not running
fi
}

force_stop() {
if status ; then
stop
status && kill -KILL `cat "$pidfile"`
fi
}


case "$1" in
start)
status
code=$?
if [ $code -eq 0 ]; then
echo "$name is already running"
else
start
code=$?
fi
exit $code
;;
stop) stop ;;
force-stop) force_stop ;;
status)
status
code=$?
if [ $code -eq 0 ] ; then
echo "$name is running"
else
echo "$name is not running"
fi
exit $code
;;
restart)

stop && start
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|restart}" >&2
exit 3
;;
esac

exit $?


[root@tzgsqvapp01 app]# chmod +x /etc/init.d/logstash
[root@tzgsqvapp01 app]# chkconfig --add logstash
[root@tzgsqvapp01 app]# chkconfig logstash on

[root@tzgsqvapp01 app]# vim /opt/app/logstash/conf.d/logstash.conf
input {
   file {
        path => "/opt/logs/tomcat7-rest/catalina.*.out"
        type => "tomcat-catalina-out"
        #codec => multiline {
        #        pattern => "%{TIMESTAMP_ISO08601} "
        #        negate => true
        #        what => previous
        #}
        add_field => {"module" => "rest" }
    }

}


output {
    if [type] == "tomcat-catalina-out" {
    redis {
        data_type => "channel"
        host => "redisIP"
        port => "6379"
        key => "tomcat-catalina-out"
        db => "0"
         }
    }
}

[root@tzgsqvapp01 app]# ln -sv /opt/app/jdk7/bin/java /usr/bin/java
[root@tzgsqvapp01 app]# systemctl daemon-reload
[root@tzgsqvapp01 app]# systemctl start logstash


[root@tzgsqvapp01 logstash]# cat logstash.log
{:timestamp=>"2017-01-13T22:59:26.174000+0800", :message=>"Pipeline main started"}
{:timestamp=>"2017-01-13T23:00:57.972000+0800", :message=>"SIGTERM received. Shutting down the agent.", :level=>:warn}
{:timestamp=>"2017-01-13T23:00:57.973000+0800", :message=>"stopping pipeline", :id=>"main"}
{:timestamp=>"2017-01-13T23:00:58.379000+0800", :message=>"Pipeline main has been shutdown"}
{:timestamp=>"2017-01-13T23:01:07.800000+0800", :message=>"Pipeline main started"}
[root@tzgsqvapp01 logstash]# ll
total 8
-rw-r--r-- 1 root     root       0 Jan 13 23:00 logstash.err
-rw-r--r-- 1 logstash logstash 470 Jan 13 23:01 logstash.log
-rw-r--r-- 1 root     root     141 Jan 13 23:01 logstash.stdout
[root@tzgsqvapp01 logstash]# cat logstash.stdout
Sending logstash logs to /opt/logs/logstash/logstash.log.
{:timestamp=>"2017-01-13T23:01:07.800000+0800", :message=>"Pipeline main started"}

ELK服務(wù)器上

[tzg@tzgsqvelk01 redis]$ bin/redis-cli 
redisIP:6379> SUBSCRIBE tomcat-catalina-out
Reading messages... (press Ctrl-C to quit)
1) "subscribe"
2) "tomcat-catalina-out"
3) (integer) 1
1) "message"
2) "tomcat-catalina-out"
3) "{\"message\":\" 2017-01-13 23:10:01.797 [DubboMonitorSendTimer-thread-1] INFO  com.alibaba.dubbo.monitor.dubbo.DubboMonitor[Slf4jLogger.java:42] [for_log_analysis_id]--[for_log_analysis_ip]  [DUBBO] Send statistics to monitor zookeeper://zk01.tzg.sq:2181/com.alibaba.dubbo.monitor.MonitorService?anyhost=true&application=simple-monitor&check=false&delay=-1&dubbo=2.5.3&interface=com.alibaba.dubbo.monitor.MonitorService&methods=lookup,collect&owner=tzg&pid=1179&revision=2.5.3&side=provider&timestamp=1484029892639, dubbo version: 2.5.3, current host: 172.16.5.13\",\"@version\":\"1\",\"@timestamp\":\"2017-01-13T15:10:02.118Z\",\"path\":\"/opt/logs/tomcat7-rest/catalina.2017-01-13.out\",\"host\":\"tzgsqvapp01\",\"type\":\"tomcat-catalina-out\",\"module\":\"rest\"}"


[root@tzgsqvelk01 app]# cat /etc/logstash/conf.d/logstash.conf
input {
    redis {
        data_type => "channel"
        key => "tomcat-catalina-out"
        host => "redisIP"
        port => 6379
        db => 0
    }
}

output {
    elasticsearch { hosts => "localhost:9200"
                    user => elastic
                    password => 123456 }
    file {
        codec => line { format => "%{message}"}
        path => "/tzgData/logstash/%{+YYYY.MM.dd}/%{module}.%{type}"
        }
}

[root@tzgsqvelk01 app]# systemctl start logstash


最后編輯于
?著作權(quán)歸作者所有,轉(zhuǎn)載或內(nèi)容合作請聯(lián)系作者
  • 序言:七十年代末,一起剝皮案震驚了整個(gè)濱河市婆排,隨后出現(xiàn)的幾起案子,更是在濱河造成了極大的恐慌,老刑警劉巖衔掸,帶你破解...
    沈念sama閱讀 218,451評論 6 506
  • 序言:濱河連續(xù)發(fā)生了三起死亡事件战虏,死亡現(xiàn)場離奇詭異策严,居然都是意外死亡,警方通過查閱死者的電腦和手機(jī)蝙寨,發(fā)現(xiàn)死者居然都...
    沈念sama閱讀 93,172評論 3 394
  • 文/潘曉璐 我一進(jìn)店門,熙熙樓的掌柜王于貴愁眉苦臉地迎上來嗤瞎,“玉大人墙歪,你說我怎么就攤上這事”雌妫” “怎么了虹菲?”我有些...
    開封第一講書人閱讀 164,782評論 0 354
  • 文/不壞的土叔 我叫張陵,是天一觀的道長掉瞳。 經(jīng)常有香客問我届惋,道長,這世上最難降的妖魔是什么菠赚? 我笑而不...
    開封第一講書人閱讀 58,709評論 1 294
  • 正文 為了忘掉前任脑豹,我火速辦了婚禮,結(jié)果婚禮上衡查,老公的妹妹穿的比我還像新娘瘩欺。我一直安慰自己,他們只是感情好,可當(dāng)我...
    茶點(diǎn)故事閱讀 67,733評論 6 392
  • 文/花漫 我一把揭開白布俱饿。 她就那樣靜靜地躺著歌粥,像睡著了一般。 火紅的嫁衣襯著肌膚如雪拍埠。 梳的紋絲不亂的頭發(fā)上失驶,一...
    開封第一講書人閱讀 51,578評論 1 305
  • 那天,我揣著相機(jī)與錄音枣购,去河邊找鬼嬉探。 笑死,一個(gè)胖子當(dāng)著我的面吹牛棉圈,可吹牛的內(nèi)容都是我干的涩堤。 我是一名探鬼主播,決...
    沈念sama閱讀 40,320評論 3 418
  • 文/蒼蘭香墨 我猛地睜開眼分瘾,長吁一口氣:“原來是場噩夢啊……” “哼胎围!你這毒婦竟也來了?” 一聲冷哼從身側(cè)響起德召,我...
    開封第一講書人閱讀 39,241評論 0 276
  • 序言:老撾萬榮一對情侶失蹤白魂,失蹤者是張志新(化名)和其女友劉穎,沒想到半個(gè)月后上岗,有當(dāng)?shù)厝嗽跇淞掷锇l(fā)現(xiàn)了一具尸體碧聪,經(jīng)...
    沈念sama閱讀 45,686評論 1 314
  • 正文 獨(dú)居荒郊野嶺守林人離奇死亡,尸身上長有42處帶血的膿包…… 初始之章·張勛 以下內(nèi)容為張勛視角 年9月15日...
    茶點(diǎn)故事閱讀 37,878評論 3 336
  • 正文 我和宋清朗相戀三年液茎,在試婚紗的時(shí)候發(fā)現(xiàn)自己被綠了逞姿。 大學(xué)時(shí)的朋友給我發(fā)了我未婚夫和他白月光在一起吃飯的照片。...
    茶點(diǎn)故事閱讀 39,992評論 1 348
  • 序言:一個(gè)原本活蹦亂跳的男人離奇死亡捆等,死狀恐怖滞造,靈堂內(nèi)的尸體忽然破棺而出,到底是詐尸還是另有隱情栋烤,我是刑警寧澤谒养,帶...
    沈念sama閱讀 35,715評論 5 346
  • 正文 年R本政府宣布,位于F島的核電站明郭,受9級特大地震影響买窟,放射性物質(zhì)發(fā)生泄漏。R本人自食惡果不足惜薯定,卻給世界環(huán)境...
    茶點(diǎn)故事閱讀 41,336評論 3 330
  • 文/蒙蒙 一始绍、第九天 我趴在偏房一處隱蔽的房頂上張望。 院中可真熱鬧话侄,春花似錦亏推、人聲如沸学赛。這莊子的主人今日做“春日...
    開封第一講書人閱讀 31,912評論 0 22
  • 文/蒼蘭香墨 我抬頭看了看天上的太陽盏浇。三九已至,卻和暖如春芽狗,著一層夾襖步出監(jiān)牢的瞬間绢掰,已是汗流浹背。 一陣腳步聲響...
    開封第一講書人閱讀 33,040評論 1 270
  • 我被黑心中介騙來泰國打工童擎, 沒想到剛下飛機(jī)就差點(diǎn)兒被人妖公主榨干…… 1. 我叫王不留滴劲,地道東北人。 一個(gè)月前我還...
    沈念sama閱讀 48,173評論 3 370
  • 正文 我出身青樓柔昼,卻偏偏與公主長得像哑芹,于是被迫代替她去往敵國和親炎辨。 傳聞我的和親對象是個(gè)殘疾皇子捕透,可洞房花燭夜當(dāng)晚...
    茶點(diǎn)故事閱讀 44,947評論 2 355

推薦閱讀更多精彩內(nèi)容