ELK-day02
image.png
input 我們要采集的日志文件路徑, 收割機 harvester 監(jiān)聽文件的變化 -->splooer程序 --> 轉(zhuǎn)發(fā) es | logstash | kafka | redis
image.png
filebeat.inputs:
- type: stdin #標準輸入
enabled: true #啟用
output.console: #標準輸出
pretty: true
enable: true
image.png
將文件最新發(fā)生變化的內(nèi)容,存入ES
[root@web01 ~]# cat /etc/filebeat/file.yml
filebeat.inputs:
- type: log
paths: /var/log/nginx/access.log
enabled: true
output.elasticsearch:
hosts:
["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
收集系統(tǒng)日志
特別分散--> syslog --> file.txt
1.減少無用的數(shù)據(jù)
2.調(diào)整索引名稱
3.測試調(diào)整模板,設定分片
[root@web01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/oldxu.log
include_lines: ['^ERR', '^WARN', 'sshd'] #只看指定的
日志
output.elasticsearch:
hosts:
["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
index: "system-%{[agent.version]}-%{+yyyy.MM.dd}"
setup.ilm.enabled: false
setup.template.name: system #索引關聯(lián)的模板名稱
setup.template.pattern: system-*
方式一:
###設定system模板的分片數(shù)和副本數(shù)
#setup.template.settings: #定義索引分片數(shù)和副本
# index.number_of_shards: 3
# index.number_of_replicas: 1
方式二:
"number_of_shards": "10",
"number_of_replicas": "1",
1.修改system模板 ---> 添加 shards 分片數(shù)數(shù)
量,replicas的數(shù)量
2.刪除模板關聯(lián)的索引
3.刪除filebeat自行指定的分片數(shù)和副本數(shù)
4.重啟filebeat
5.產(chǎn)生新的日志
image.png
收集Nginx
配置filebeat
log_format json '{ "time_local": "$time_local", '
'"remote_addr":
"$remote_addr", '
'"referer": "$http_referer",
'
'"request": "$request", '
'"status": $status, '
'"bytes": $body_bytes_sent, '
'"agent": "$http_user_agent",
'
'"x_forwarded":
"$http_x_forwarded_for", '
'"up_addr":
"$upstream_addr",'
'"up_host":
"$upstream_http_host",'
'"upstream_time":
"$upstream_response_time",'
'"request_time":
"$request_time"'
'}';
access_log /var/log/nginx/access.log json;
1 [root@web01 filebeat]# cat /etc/filebeat/filebeat.yml
收集nginx訪問日志和錯誤日志
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true #默認Flase雌贱，還會將json解析
的日志存儲至messages字段
json.overwrite_keys: true #覆蓋默認的key丰刊，使用自定義
json格式的key
output.elasticsearch:
hosts:
["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
setup.ilm.enabled: false
setup.template.name: nginx #索引關聯(lián)的模板名稱
setup.template.pattern: nginx-*
[root@web01 filebeat]# cat filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true #默認Flase志笼，還會將json解析
的日志存儲至messages字段
json.overwrite_keys: true #覆蓋默認的key，使用自定義
json格式的key
tags: ["access"]
收集nginx多個虛擬主機的日志
elk.oldxu.com
bk.oldxu.com
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["error"]
output.elasticsearch:
hosts:
["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
indices:
- index: "nginx-access-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "access"
- index: "nginx-error-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "error"
setup.ilm.enabled: false
setup.template.name: nginx #索引關聯(lián)的模板名稱
setup.template.pattern: nginx-*
image.png
bs.oldxu.com
error日志
1.虛擬主機
[root@web01 conf.d]# cat elk.oldxu.com.conf
server {
listen 80;
server_name elk.oldxu.com;
root /code/elk;
access_log /var/log/nginx/elk.oldxu.com.log json;
location / {
index index.html;
}
}
[root@web01 conf.d]# cat bs.oldxu.com.conf
server {
listen 80;
server_name bs.oldxu.com;
2.測試,模擬產(chǎn)生日志
3.配置filebeat
root /code/bs;
access_log /var/log/nginx/bs.oldxu.com.log json;
location / {
index index.html;
}
}
[root@web01 conf.d]# cat bk.oldxu.com.conf
server {
listen 80;
server_name bk.oldxu.com;
root /code/bk;
access_log /var/log/nginx/bk.oldxu.com.log json;
location / {
index index.html;
}
}
[root@web01 conf.d]# curl -H Host:elk.oldxu.com
http://10.0.0.7
elk.oldux.com
[root@web01 conf.d]# curl -H Host:bs.oldxu.com
http://10.0.0.7
bs.oldux.com
[root@web01 conf.d]# curl -H Host:bk.oldxu.com
http://10.0.0.7
bk.oldux.com
[root@web01 filebeat]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/elk.oldxu.com.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["nginx-elk-host"]
- type: log
enabled: true
paths:
- /var/log/nginx/bs.oldxu.com.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["nginx-bs-host"]
- type: log
enabled: true
paths:
- /var/log/nginx/bk.oldxu.com.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["nginx-bk-host"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["nginx-error"]
Tomcat日志
訪問日志 ---> json格式
output.elasticsearch:
hosts:
["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
indices:
- index: "nginx-elk-access-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "nginx-elk-host"
- index: "nginx-bs-access-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "nginx-bs-host"
- index: "nginx-bk-access-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "nginx-bk-host"
- index: "nginx-error-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "nginx-error"
setup.ilm.enabled: false
setup.template.name: nginx #索引關聯(lián)的模板名稱
setup.template.pattern: nginx-*
配置filebeat
#1.修改tomcat日志格式
[root@web02 soft]# yum install java -y
[root@web02 soft]# vim tomcat/conf/server.xml
<Host name="tomcat.oldxu.com" appBase="webapps"
unpackWARs="true" autoDeploy="true">
<Valve
className="org.apache.catalina.valves.AccessLogValve"
directory="logs"
prefix="tomcat.oldxu.com.log"
suffix=".txt"
pattern="
{"clientip":"%h","ClientUser&q
uot;:"%l","authenticated":"%u&
quot;,"AccessTime":"%t","metho
d":"%r","status":"%s"
;,"SendBytes":"%b","Query?
string":"%q","partner":"%
{Referer}i","AgentVersion":"%{UserAgent}i"}" />
</Host>
[root@web02 filebeat]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /soft/tomcat/logs/tomcat.oldxu.com.log.*.txt
json.keys_under_root: true #默認Flase霉猛，還會將json解析
的日志存儲至messages字段
json.overwrite_keys: true #覆蓋默認的key邪财，使用自定義
json格式的key
錯誤日志 <--java
output.elasticsearch:
hosts: ["10.0.0.161:9200","10.0.0.162:9200"]
index: "tomcat-access-%{[agent.version]}-%
{+yyyy.MM.dd}"
setup.ilm.enabled: false
setup.template.name: tomcat #索引關聯(lián)的模板名稱
setup.template.pattern: tomcat-*
[root@web02 filebeat]# cat filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /soft/tomcat/logs/tomcat.oldxu.com.log.*.txt
json.keys_under_root: true #默認Flase啥刻，還會將json解析
的日志存儲至messages字段
json.overwrite_keys: true #覆蓋默認的key煤搜，使用自定義
json格式的key
tags: ["tomcat-access"]
- type: log
enabled: true
paths:
- /soft/tomcat/logs/catalina.out
multiline.pattern: '^\d{2}' #匹配以2個數(shù)字開頭的
multiline.negate: true
multiline.match: after
multiline.max_lines: 10000 #默認最大合并行為500，可根
據(jù)實際情況調(diào)整凡伊。
tags: ["tomcat-error"]
output.elasticsearch:
hosts: ["10.0.0.161:9200","10.0.0.162:9200"]
indices:
- index: "tomcat-access-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "tomcat-access"
- index: "tomcat-error-%{[agent.version]}-%
{+yyyy.MM.dd}"
when.contains:
tags: "tomcat-error"
setup.ilm.enabled: false
setup.template.name: tomcat #索引關聯(lián)的模板名稱
setup.template.pattern: tomcat-*
