一共啃、配置Logstash
input {
s3 {
aws_credentials_file => "/app/aws-credentials.yaml"
region => "us-west-1"
bucket => "aws-test-qa"
prefix => "QA_APILOG_2021-04/ip-10-0-1-10/2021-04-14"
sincedb_path => "/app/logstash/.qa-kong-logs-s3-sincedb"
type => "qa-kong-logs-s3"
}
}
filter {
if [type] == "qa-kong-logs-s3" {
grok {
match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:msg_content}" }
}
json {
source => "msg_content"
remove_field => ["message","msg_content"]
}
date {
match => [ "timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss"]
target => "@timestamp"
remove_field => [ "timestamp" ]
}
if "_grokparsefailure" in [tags] {
drop { }
}
}
}
output {
if [type] == "qa-kong-logs-s3" {
elasticsearch {
hosts => ["localhost:9200"]
user => "qa"
password => "xxxxx"
index => "qa-kong-logs-s3-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}
}
二占调、測試配置
sudo /usr/share/logstash/bin/logstash -t -f /etc/logstash/conf.d/qa-kong-logs-s3.conf
幾秒鐘后，如果沒有語法錯誤移剪，它應(yīng)該會顯示Configuration OK究珊。否則，嘗試讀取錯誤輸出纵苛，查看Logstash配置有什么問題
三剿涮、重啟Logstash
sudo systemctl restart logstash
sudo systemctl enable logstash
四、檢查Elasticsearch中的索引
curl http://localhost:9200/_cat/indices/qa-kong-logs-s3*
