Тема: Elasticsearch Logstash Kibana (NON-AWS)

В продолжение к Elasticsearch Logstash Kibana (AWS). Все таки сделал свой лог сервер.

elasticsearch

apt-get install software-properties-common
add-apt-repository "deb http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main"
apt-get update
apt-get install oracle-java8-installer
wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add -
echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
apt-get update
apt-get install elasticsearch
vi /etc/elasticsearch/elasticsearch.yml
systemctl enable elasticsearch
systemctl restart elasticsearch

kibana

echo "deb http://packages.elastic.co/kibana/4.5/debian stable main" | sudo tee -a /etc/apt/sources.list.d/kibana-4.5.x.list
apt-get update
apt-get install kibana
vi /opt/kibana/config/kibana.yml
systemctl enable kibana
systemctl restart kibana

nginx

apt-get install nginx
vi /etc/nginx/sites-available/default
/etc/init.d/nginx configtest
systemctl enable nginx
systemctl restart nginx
...
    location / {
        proxy_pass http://localhost:5601;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;        
    }
...

logstash

echo 'deb http://packages.elastic.co/logstash/2.2/debian stable main' | sudo tee /etc/apt/sources.list.d/logstash-2.2.x.list
apt-get update
apt-get install logstash
vi /etc/logstash/conf.d/02-beats-input.conf
vi /etc/logstash/conf.d/10-syslog-filter.conf
vi /etc/logstash/conf.d/30-elasticsearch-output.conf
/etc/init.d/logstash configtest
systemctl enable logstash
systemctl restart logstash

Пару примеров как принимать данные
/etc/logstash/conf.d/10-input-syslog.conf

input {
  tcp {
    port => 1514
    type => "syslog"
    codec => "json"
  }
  udp {
    port => 1514
    type => "syslog"
    codec => "json"
  }
}

/etc/logstash/conf.d/30-filter-haproxy.conf

filter {
  if [programname] == "haproxy" {
    mutate {.
      replace => { "type" => "haproxy" }
    }
  }
}

/etc/logstash/patterns/nginx

NGINX_ACCESS %{IPORHOST:clientip} (?:-|(%{WORD}.%{WORD})) %{USER:ident} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:forwarder}

/etc/logstash/conf.d/30-filter-nginx.conf

filter {
  if [programname] == "nginx" {
    grok {
      patterns_dir => "/etc/logstash/patterns"
      match => {
        "message" => "%{NGINX_ACCESS}"
      }
      remove_tag => ["_grokparsefailure"]
      remove_field => ["message","port","facility"]
    }
    mutate {
      replace => {
        "type" => "nginx"
      }
    }
    useragent {
      source => "agent"
    }
    geoip {
      source => "clientip"
      target => "geoip"
      add_tag => ["geoip"]
    }
  }
}

/etc/logstash/conf.d/30-filter-syslog.conf

filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }
}

/etc/logstash/conf.d/60-output-elasticsearch.conf

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "%{[type]}-%{+YYYY.MM.dd}"
    sniffing => true
  }
}

Ну и пример темалайта для nginx чтобы использовать geoip
nginx-template.json

{
    "order": 0,
    "template": "nginx-*",
    "settings": {
        "index": {
            "refresh_interval": "5s"
        }
    },
    "mappings": {
        "_default_": {
            "dynamic_templates": [{
                    "message_field": {
                        "mapping": {
                            "fielddata": {
                                "format": "disabled"
                            },
                            "index": "analyzed",
                            "omit_norms": true,
                            "type": "string"
                        },
                        "match_mapping_type": "string",
                        "match": "message"
                    }
                },
                {
                    "string_fields": {
                        "mapping": {
                            "fielddata": {
                                "format": "disabled"
                            },
                            "index": "analyzed",
                            "omit_norms": true,
                            "type": "string",
                            "fields": {
                                "raw": {
                                    "ignore_above": 256,
                                    "index": "not_analyzed",
                                    "type": "string",
                                    "doc_values": true
                                }
                            }
                        },
                        "match_mapping_type": "string",
                        "match": "*"
                    }
                },
                {
                    "float_fields": {
                        "mapping": {
                            "type": "float",
                            "doc_values": true
                        },
                        "match_mapping_type": "float",
                        "match": "*"
                    }
                },
                {
                    "double_fields": {
                        "mapping": {
                            "type": "double",
                            "doc_values": true
                        },
                        "match_mapping_type": "double",
                        "match": "*"
                    }
                },
                {
                    "byte_fields": {
                        "mapping": {
                            "type": "byte",
                            "doc_values": true
                        },
                        "match_mapping_type": "byte",
                        "match": "*"
                    }
                },
                {
                    "short_fields": {
                        "mapping": {
                            "type": "short",
                            "doc_values": true
                        },
                        "match_mapping_type": "short",
                        "match": "*"
                    }
                },
                {
                    "integer_fields": {
                        "mapping": {
                            "type": "integer",
                            "doc_values": true
                        },
                        "match_mapping_type": "integer",
                        "match": "*"
                    }
                },
                {
                    "long_fields": {
                        "mapping": {
                            "type": "long",
                            "doc_values": true
                        },
                        "match_mapping_type": "long",
                        "match": "*"
                    }
                },
                {
                    "date_fields": {
                        "mapping": {
                            "type": "date",
                            "doc_values": true
                        },
                        "match_mapping_type": "date",
                        "match": "*"
                    }
                },
                {
                    "geo_point_fields": {
                        "mapping": {
                            "type": "geo_point",
                            "doc_values": true
                        },
                        "match_mapping_type": "geo_point",
                        "match": "*"
                    }
                }
            ],
            "_all": {
                "omit_norms": true,
                "enabled": true
            },
            "properties": {
                "@timestamp": {
                    "type": "date",
                    "doc_values": true
                },
                "geoip": {
                    "dynamic": true,
                    "type": "object",
                    "properties": {
                        "ip": {
                            "type": "ip",
                            "doc_values": true
                        },
                        "latitude": {
                            "type": "float",
                            "doc_values": true
                        },
                        "location": {
                            "type": "geo_point",
                            "doc_values": true
                        },
                        "longitude": {
                            "type": "float",
                            "doc_values": true
                        }
                    }
                },
                "@version": {
                    "index": "not_analyzed",
                    "type": "string",
                    "doc_values": true
                }
            }
        }
    },
    "aliases": {}
}

и как его закачать

curl -XPUT 'http://localhost:9200/_template/nginx?pretty' -d@nginx-template.json

Как слать данные напрямую с rsyslogd. Все он умеет как оказалось.
/etc/rsyslog.d/nginx.conf

module(load="imfile" PollingInterval="10") #needs to be done just once

input(
  type="imfile"
  file="/var/log/nginx/api_access.log"
  statefile="api_access.log.state"
  tag="api_access.log"
  facility="local7"
)

input(
  type="imfile"
  file="/var/log/nginx/api-v1_access.log"
  statefile="api-v1_access.log"
  tag="api-v1_access.log"
  facility="local7"
)

template(name="jt-nginx" type="list") {
  constant(value="{")
  constant(value="\"@timestamp\":\"")          property(name="timereported" dateFormat="rfc3339")
  constant(value="\",\"@version\":\"1")
  constant(value="\",\"logformat\":\"1")
  constant(value="\",\"message\":\"")          property(name="msg" format="json")
  constant(value="\",\"sysloghost\":\"")       property(name="hostname")
  constant(value="\",\"severity\":\"")         property(name="syslogseverity-text")
  constant(value="\",\"facility\":\"")         property(name="syslogfacility-text")
  constant(value="\",\"programname\":\"nginx")
  constant(value="\",\"file_name\":\"")        property(name="programname")
  constant(value="\",\"procid\":\"")           property(name="procid")
  constant(value="\"}\n")
}

if ($syslogfacility-text == 'local7') then {
  action(type="omfwd" target="111.111.111.111" port="1514" protocol="tcp" template="jt-nginx")
  &stop
}

/etc/rsyslog.d/haproxy.conf

$AddUnixListenSocket /var/lib/haproxy/dev/log

template(name="jt-haproxy" type="list") {
  constant(value="{")
  constant(value="\"@timestamp\":\"")     property(name="timereported" dateFormat="rfc3339")
  constant(value="\",\"@version\":\"1")
  constant(value="\",\"logformat\":\"2")
  constant(value="\",\"message\":\"")     property(name="msg" format="json")
  constant(value="\",\"sysloghost\":\"")  property(name="hostname")
  constant(value="\",\"severity\":\"")    property(name="syslogseverity-text")
  constant(value="\",\"facility\":\"")    property(name="syslogfacility-text")
  constant(value="\",\"programname\":\"") property(name="programname")
  constant(value="\",\"procid\":\"")      property(name="procid")
  constant(value="\"}\n")
}

if ($programname startswith "haproxy") then {
  action(type="omfile" file="/var/log/haproxy.log")
  action(type="omfwd" target="111.111.111.111" port="1514" protocol="tcp" template="jt-haproxy")
  &stop
}

Что получается, шлем данные json'ом, не храним того что не надо, в nginx удаляется message поле, каждый сервис идет в свой отдельный index что позволяет быстрее работать с данными и удалять их в разное время...