logstashnginxjsones（6）

安装nginx

yum -y install pcre pcre-devel openss-devel
http://nginx.org/download/nginx-1.6.3.tar.gz
groupadd -r nginx
useradd -g nginx -r nginx
ln -s /usr/local/nginx-1.6.3 /usr/local/nginx

编译

 ./configure 
--prefix=/usr/local/nginx 
--conf-path=/etc/nginx/nginx.conf 
--user=nginx --group=nginx 
--error-log-path=/var/log/nginx/error.log 
--http-log-path=/var/log/nginx/access.log 
--pid-path=/var/run/nginx/nginx.pid 
--lock-path=/var/lock/nginx.lock 
--with-http_ssl_module 
--with-http_stub_status_module 
--with-http_gzip_static_module 
--with-http_flv_module 
--with-http_mp4_module 
--http-client-body-temp-path=/var/tmp/nginx/client 
--http-proxy-temp-path=/var/tmp/nginx/proxy 
--http-fastcgi-temp-path=/var/tmp/nginx/fastcgi 
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi
make && make install
mkdir -pv /var/tmp/nginx/{client,fastcgi,proxy,uwsgi}
mkdir /usr/local/nginx/logs/
/usr/local/sbin/nginx

编辑nginx配置文件：

vim /etc/nginx/nginx.conf

添加如下字段：

#access_log  logs/access.log  main;
log_format logstash_json '{"@timestamp":"$time_iso8601",'
     '"host": "$server_addr",'
     '"client": "$remote_addr",'
     '"size": $body_bytes_sent,'
     '"responsetime": $request_time,'
     '"domain": "$host",'
     '"url":"$uri",'
     '"referer": "$http_referer",'
     '"agent": "$http_user_agent",'
     '"status":"$status"}';

修改如下：

 access_log  logs/access_json.access.log  logstash_json;

访问后测试：

[root@elk1 logs]# ab -n1000 -c10 http://192.168.1.4:81/

查看日志

[root@elk1 nginx]# cat /usr/local/nginx/logs/access_json.access.log 
{"@timestamp":"2016-03-20T05:46:57-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 612,"responsetime": 0.000,"domain": "192.168.1.4","url":"/index.html","referer": "-","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"200"}
{"@timestamp":"2016-03-20T05:46:57-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 570,"responsetime": 0.000,"domain": "192.168.1.4","url":"/favicon.ico","referer": "http://192.168.1.4:81/","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"404"}
{"@timestamp":"2016-03-20T05:46:59-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 0,"responsetime": 0.000,"domain": "192.168.1.4","url":"/index.html","referer": "-","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"304"}
{"@timestamp":"2016-03-20T05:46:59-07:00","host": "192.168.1.4","client": "192.168.1.3","size": 0,"responsetime": 0.000,"domain": "192.168.1.4","url":"/index.html","referer": "-","agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36","status":"304"}
[root@elk1 nginx]# 

产生一些日志让logstash收集

[root@elk1 nginx]# ab -n1000 -c10 http://192.168.1.4:81/
[root@elk1 logs]# ll
total 440
-rw-r--r-- 1 root root 449286 Mar 20 05:57 access_json.access.log
[root@elk1 logs]# 

当测试日志可用后，修改logstash配置文件，将access_json.access.log推到redis

[root@elk1 logs]# cat /etc/logstash.conf 
input {
# file {
#    path => "/var/log/messages"
#    type => "system-log"
#    }
 file {
        path => "/usr/local/nginx/logs/access_json.access.log"
    codec => "json"        
}
}
output {
#  redis {
#    host => "192.168.1.6"
#    data_type => "list"
#    key => "system.messages"
#    port => "6379"
#    db => "1"
#}
  redis {
    host => "192.168.1.6"
    data_type => "list"
    key => "nginx-access.log"
    port => "6379"
    db => "2"
}
}
[root@elk1 logs]# 

而后在模拟一些日志

[root@elk1 logs]# ab -n1000 -c10 http://192.168.1.4:81/

而后在redis上查看是否传递到redis

redis 192.168.1.6:6379[2]> select 2
OK
redis 192.168.1.6:6379[2]> keys *
1) "nginx-access.log"
redis 192.168.1.6:6379[2]> llen nginx-access.log
(integer) 1000
redis 192.168.1.6:6379[2]> 

验证数据存在，修改logstash文件传递到es，配置如下：

[root@yum-down ~]# cat /etc/logstash.conf 
input {
#    redis {
#        host => "192.168.1.6"
#        data_type => "list"
#        key => "test.log"
 #       port => "6379"
#        db => "1"
#}
    redis {
        host => "192.168.1.6"
        data_type => "list"
        key => "nginx-access.log"  #key名称和redis保持一致
        port => "6379"
        db => "2"    #db2
}
}
output {
#   elasticsearch {
#    host => ["192.168.1.4:9200","192.168.1.5:9200"]
#    index => "redis-system-messages-%{+YYYY.MM.dd.HH}"
#        protocol => "http"
#        workers => 5
#        template_overwrite => true
#    }
   elasticsearch {
    host => ["192.168.1.4:9200","192.168.1.5:9200"]
    index => "nginx-access-log-%{+YYYY.MM.dd.HH}"  #修改es中日志名称
        protocol => "http"
        workers => 5
        template_overwrite => true
}
}
[root@yum-down ~]#