ELK5.5Haproxy日志grok处理(filebeat)

haproxy本身的话是不建议rsyslog的日志，一般都是关闭的，但是我觉得可以用elk做一些切割尝试。我们先安装编译安装参考：https://www.linuxea.com/1328.html在后面的filebeat中会exclude_lines过滤掉["started","Pausing","Enabling","DOWN","UP","admin_stats","backend"]的信息，日志最终收集的效果如下：

日志开启

修改rsyslog.conf配置文件如下：

$ModLoad imudp
$UDPServerRun 514
local3.*                                              /var/log/haproxy.log

注释掉 #*.info;mail.none;authpriv.none;cron.none /var/log/messages 添加 *.info;mail.none;authpriv.none;cron.none;local3.none /var/log/messages

修改rsyslog

[root@LinuxEA haproxy]# vim /etc/sysconfig/rsyslog 
SYSLOGD_OPTIONS="-r -m  0 -c 2"
[root@LinuxEA haproxy]# systemctl restart rsyslog.service

日志格式是这样的：

2017-09-07T14:19:41+08:00 localhost haproxy[32171]: 10.10.0.96:50482 [07/Sep/2017:14:19:41.179] frontend-web.com linuxea-webgroup.com/<NOSRV> 0/-1/-1/-1/0 503 212 - - SC-- 0/0/0/0/0 0/0 "GET /favicon.ico HTTP/1.1"

filebeat配置文件

[root@linuxea.com-Node117 /data/logs]# cat /etc/filebeat/filebeat.yml 
filebeat.prospectors:
 - input_type: log
   paths:
    - /data/logs/access_nginx.log
   document_type: nginx-access-117
 - input_type: log
   paths:
    - /data/logs/slow_log.CSV
   document_type: mysql-slow-117
 - input_type: log
   paths:
    - /data/logs/redis_6379.log
   document_type: redis-6379-117
   include_lines: ["WARNING","ERR"]
 - input_type: log
   paths:
     - /data/logs/haproxy.log
   exclude_lines: ["started","Pausing","Enabling","DOWN","UP","admin_stats","backend"]
   document_type: haproxy-117
output.redis:
  hosts: ["10.10.0.98"]
  password: "OTdmOWI4ZTM4NTY1M2M4OTZh"
  key: "default_list"
  db: 5
  timeout: 5
  keys:
    - key: "%{[type]}"
      mapping:
      "nginx-access-117": "nginx-access-117"
      "mysql-slow-117" : "mysql-slow-117"
      "redis-6379-117" : "redis-6379-117"
      "haproxy-117" : "haproxy-117"
[root@linuxea.com-Node117 /data/logs]# 

Logstash配置

路径下有很多自带的patterns：

/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.1.1/patterns
[root@linuxea.com-Node49 /usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.1.1/patterns]# ll
总用量 112
-rw-r--r-- 1 logstash logstash  1831 7月  19 05:15 aws
-rw-r--r-- 1 logstash logstash  4831 7月  19 05:15 bacula
-rw-r--r-- 1 logstash logstash   260 7月  19 05:15 bind
-rw-r--r-- 1 logstash logstash  2154 7月  19 05:15 bro
-rw-r--r-- 1 logstash logstash   879 7月  19 05:15 exim
-rw-r--r-- 1 logstash logstash 10095 7月  19 05:15 firewalls
-rw-r--r-- 1 logstash logstash  5338 7月  19 05:15 grok-patterns
-rw-r--r-- 1 logstash logstash  3251 7月  19 05:15 haproxy
-rw-r--r-- 1 logstash logstash   987 7月  19 05:15 httpd
-rw-r--r-- 1 logstash logstash  1265 7月  19 05:15 java
-rw-r--r-- 1 logstash logstash  1087 7月  19 05:15 junos
-rw-r--r-- 1 logstash logstash  1037 7月  19 05:15 linux-syslog
-rw-r--r-- 1 logstash logstash    74 7月  19 05:15 maven
-rw-r--r-- 1 logstash logstash    49 7月  19 05:15 mcollective
-rw-r--r-- 1 logstash logstash   190 7月  19 05:15 mcollective-patterns
-rw-r--r-- 1 logstash logstash   614 7月  19 05:15 mongodb
-rw-r--r-- 1 logstash logstash  9597 7月  19 05:15 nagios
-rw-r--r-- 1 logstash logstash   142 7月  19 05:15 postgresql
-rw-r--r-- 1 logstash logstash   845 7月  19 05:15 rails
-rw-r--r-- 1 logstash logstash   224 7月  19 05:15 redis
-rw-r--r-- 1 logstash logstash   188 7月  19 05:15 ruby
-rw-r--r-- 1 logstash logstash   404 7月  19 05:15 squid
[root@linuxea.com-Node49 /usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.1.1/patterns]# 

也可以自己写到定义的位置,如：patterns_dir => ["/etc/logstash/patterns.d"]

[root@linuxea.com-Node49 /etc/logstash/patterns.d]# cat ../conf.d/redis-output.yml 
input {
    redis {
         host => "10.10.0.98"
         port => "6379"
         key => "haproxy-117"
         data_type => "list"
         password => "OTdmOWI4ZTM4NTY1M2M4OTZh"
         threads => "5"
         db => "5"
       }
    }
filter {
 if [type] == "haproxy-117" {
     grok {
#          patterns_dir => ["/etc/logstash/patterns.d"]
          match => ["message",  "%{HAPROXYHTTP}"]
          }
     date {
          match => ["accept_date", "dd/MMM/yyyy:HH:mm:ss.SSS"]
          }
    geoip {
        source => "client_ip"
        database => "/etc/logstash/GeoLite2-City.mmdb"
        }
    }
}
output {
    if "_grokparsefailure" in [tags] {
    file { path => "/var/log/logstash/grokparsefailure-%{[type]}-%{+YYYY.MM.dd}.log" }
    }
    if [type] == "haproxy-117" {
    elasticsearch {
         hosts => ["10.0.1.49:9200"]
         index => "logstash-haproxy-117-%{+YYYY.MM.dd}"
         user => "elastic"
         password => "linuxea"
     }
     }
    stdout {codec => rubydebug}
}
[root@linuxea.com-Node49 /etc/logstash/patterns.d]#