Тема: Elasticsearch Logstash Kibana (NON-AWS)
В продолжение к Elasticsearch Logstash Kibana (AWS). Все таки сделал свой лог сервер.
elasticsearch
apt-get install software-properties-common
add-apt-repository "deb http://ppa.launchpad.net/webupd8team/java/ubuntu xenial main"
apt-get update
apt-get install oracle-java8-installer
wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add -
echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
apt-get update
apt-get install elasticsearch
vi /etc/elasticsearch/elasticsearch.yml
systemctl enable elasticsearch
systemctl restart elasticsearchkibana
echo "deb http://packages.elastic.co/kibana/4.5/debian stable main" | sudo tee -a /etc/apt/sources.list.d/kibana-4.5.x.list
apt-get update
apt-get install kibana
vi /opt/kibana/config/kibana.yml
systemctl enable kibana
systemctl restart kibananginx
apt-get install nginx
vi /etc/nginx/sites-available/default
/etc/init.d/nginx configtest
systemctl enable nginx
systemctl restart nginx...
location / {
proxy_pass http://localhost:5601;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
...logstash
echo 'deb http://packages.elastic.co/logstash/2.2/debian stable main' | sudo tee /etc/apt/sources.list.d/logstash-2.2.x.list
apt-get update
apt-get install logstash
vi /etc/logstash/conf.d/02-beats-input.conf
vi /etc/logstash/conf.d/10-syslog-filter.conf
vi /etc/logstash/conf.d/30-elasticsearch-output.conf
/etc/init.d/logstash configtest
systemctl enable logstash
systemctl restart logstashПару примеров как принимать данные
/etc/logstash/conf.d/10-input-syslog.conf
input {
tcp {
port => 1514
type => "syslog"
codec => "json"
}
udp {
port => 1514
type => "syslog"
codec => "json"
}
}/etc/logstash/conf.d/30-filter-haproxy.conf
filter {
if [programname] == "haproxy" {
mutate {.
replace => { "type" => "haproxy" }
}
}
}/etc/logstash/patterns/nginx
NGINX_ACCESS %{IPORHOST:clientip} (?:-|(%{WORD}.%{WORD})) %{USER:ident} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:forwarder}/etc/logstash/conf.d/30-filter-nginx.conf
filter {
if [programname] == "nginx" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => {
"message" => "%{NGINX_ACCESS}"
}
remove_tag => ["_grokparsefailure"]
remove_field => ["message","port","facility"]
}
mutate {
replace => {
"type" => "nginx"
}
}
useragent {
source => "agent"
}
geoip {
source => "clientip"
target => "geoip"
add_tag => ["geoip"]
}
}
}/etc/logstash/conf.d/30-filter-syslog.conf
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}/etc/logstash/conf.d/60-output-elasticsearch.conf
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "%{[type]}-%{+YYYY.MM.dd}"
sniffing => true
}
}Ну и пример темалайта для nginx чтобы использовать geoip
nginx-template.json
{
"order": 0,
"template": "nginx-*",
"settings": {
"index": {
"refresh_interval": "5s"
}
},
"mappings": {
"_default_": {
"dynamic_templates": [{
"message_field": {
"mapping": {
"fielddata": {
"format": "disabled"
},
"index": "analyzed",
"omit_norms": true,
"type": "string"
},
"match_mapping_type": "string",
"match": "message"
}
},
{
"string_fields": {
"mapping": {
"fielddata": {
"format": "disabled"
},
"index": "analyzed",
"omit_norms": true,
"type": "string",
"fields": {
"raw": {
"ignore_above": 256,
"index": "not_analyzed",
"type": "string",
"doc_values": true
}
}
},
"match_mapping_type": "string",
"match": "*"
}
},
{
"float_fields": {
"mapping": {
"type": "float",
"doc_values": true
},
"match_mapping_type": "float",
"match": "*"
}
},
{
"double_fields": {
"mapping": {
"type": "double",
"doc_values": true
},
"match_mapping_type": "double",
"match": "*"
}
},
{
"byte_fields": {
"mapping": {
"type": "byte",
"doc_values": true
},
"match_mapping_type": "byte",
"match": "*"
}
},
{
"short_fields": {
"mapping": {
"type": "short",
"doc_values": true
},
"match_mapping_type": "short",
"match": "*"
}
},
{
"integer_fields": {
"mapping": {
"type": "integer",
"doc_values": true
},
"match_mapping_type": "integer",
"match": "*"
}
},
{
"long_fields": {
"mapping": {
"type": "long",
"doc_values": true
},
"match_mapping_type": "long",
"match": "*"
}
},
{
"date_fields": {
"mapping": {
"type": "date",
"doc_values": true
},
"match_mapping_type": "date",
"match": "*"
}
},
{
"geo_point_fields": {
"mapping": {
"type": "geo_point",
"doc_values": true
},
"match_mapping_type": "geo_point",
"match": "*"
}
}
],
"_all": {
"omit_norms": true,
"enabled": true
},
"properties": {
"@timestamp": {
"type": "date",
"doc_values": true
},
"geoip": {
"dynamic": true,
"type": "object",
"properties": {
"ip": {
"type": "ip",
"doc_values": true
},
"latitude": {
"type": "float",
"doc_values": true
},
"location": {
"type": "geo_point",
"doc_values": true
},
"longitude": {
"type": "float",
"doc_values": true
}
}
},
"@version": {
"index": "not_analyzed",
"type": "string",
"doc_values": true
}
}
}
},
"aliases": {}
}и как его закачать
curl -XPUT 'http://localhost:9200/_template/nginx?pretty' [email protected]Как слать данные напрямую с rsyslogd. Все он умеет как оказалось.
/etc/rsyslog.d/nginx.conf
module(load="imfile" PollingInterval="10") #needs to be done just once
input(
type="imfile"
file="/var/log/nginx/api_access.log"
statefile="api_access.log.state"
tag="api_access.log"
facility="local7"
)
input(
type="imfile"
file="/var/log/nginx/api-v1_access.log"
statefile="api-v1_access.log"
tag="api-v1_access.log"
facility="local7"
)
template(name="jt-nginx" type="list") {
constant(value="{")
constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"@version\":\"1")
constant(value="\",\"logformat\":\"1")
constant(value="\",\"message\":\"") property(name="msg" format="json")
constant(value="\",\"sysloghost\":\"") property(name="hostname")
constant(value="\",\"severity\":\"") property(name="syslogseverity-text")
constant(value="\",\"facility\":\"") property(name="syslogfacility-text")
constant(value="\",\"programname\":\"nginx")
constant(value="\",\"file_name\":\"") property(name="programname")
constant(value="\",\"procid\":\"") property(name="procid")
constant(value="\"}\n")
}
if ($syslogfacility-text == 'local7') then {
action(type="omfwd" target="111.111.111.111" port="1514" protocol="tcp" template="jt-nginx")
&stop
}/etc/rsyslog.d/haproxy.conf
$AddUnixListenSocket /var/lib/haproxy/dev/log
template(name="jt-haproxy" type="list") {
constant(value="{")
constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"@version\":\"1")
constant(value="\",\"logformat\":\"2")
constant(value="\",\"message\":\"") property(name="msg" format="json")
constant(value="\",\"sysloghost\":\"") property(name="hostname")
constant(value="\",\"severity\":\"") property(name="syslogseverity-text")
constant(value="\",\"facility\":\"") property(name="syslogfacility-text")
constant(value="\",\"programname\":\"") property(name="programname")
constant(value="\",\"procid\":\"") property(name="procid")
constant(value="\"}\n")
}
if ($programname startswith "haproxy") then {
action(type="omfile" file="/var/log/haproxy.log")
action(type="omfwd" target="111.111.111.111" port="1514" protocol="tcp" template="jt-haproxy")
&stop
}Что получается, шлем данные json'ом, не храним того что не надо, в nginx удаляется message поле, каждый сервис идет в свой отдельный index что позволяет быстрее работать с данными и удалять их в разное время...