億級日志log4j2接入Kafka方案
背景描述
一個(gè)高訪問量高并發(fā)高響應(yīng)的系統(tǒng)干签，每天大概5~6億的請求日志量，日志接入大數(shù)據(jù)平臺(tái)，第一版使用的是Flume方式，后來因?yàn)樾枰谌罩咀鰧?shí)時(shí)預(yù)警功能辰企，故需要實(shí)時(shí)接入，采用log4j2 KafkaAppender接入Kafka消息隊(duì)列的模式况鸣。
log4j2配置
<Kafka name="kafkaLog" topic="topic_request_log" ignoreExceptions="false">
<PatternLayout pattern="[%-4level]_|_%d{YYYY-MM-dd HH:mm:ss}_|_%m_|_${sys:ip}"/>
<Property name="bootstrap.servers">bigdata001.dns.org:9092,bigdata002.dns.org:9092</Property>
<Property name="max.block.ms">2000</Property>
</Kafka>
<RollingFile name="failoverKafkaLog" fileName="../log/Service/failoverKafka/request.log"
filePattern="../log/Service/failoverKafka/request.%d{yyyy-MM-dd}.log">
<ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout>
<Pattern>[%-4level]_|_%d{YYYY-MM-dd HH:mm:ss}_|_%m_|_${sys:ip}%n</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy />
</Policies>
</RollingFile>
<Failover name="Failover" primary="kafkaLog" retryIntervalSeconds="600">
<Failovers>
<AppenderRef ref="failoverKafkaLog"/>
</Failovers>
</Failover>
<!--異步-->
<AsyncLogger name="kafkaLogger" level="INFO" additivity="false">
<appender-ref ref="Failover"/>
</AsyncLogger>
