From e4bab5b99600e28d55febef52e7d09bf1d93d8ff Mon Sep 17 00:00:00 2001 From: Pablo Castorino Date: Thu, 29 Sep 2016 22:52:05 +0200 Subject: [PATCH] add ELK support (#331) * add support to forward logs to ELK stack. * from docker elk customize image with * https://github.com/whyscream/postfix-grok-patterns * custom imput * override syslog filter. * fix typo. * Explicit forwarder vars and messages. * add amavis grok * add dovecot grok * add geoip db * add logstash geoip plugin * add custom amavis grok from @tomav. * switch to filebeats input * refactor syslog filter * add filebeat * add template config * replace rsyslog with filebeat. --- Dockerfile | 9 ++++++++- docker-compose.elk.yml.dist | 34 ++++++++++++++++++++++++++++++++++ elk/02-beats-input.conf | 6 ++++++ elk/10-syslog.conf | 14 ++++++++++++++ elk/Dockerfile | 26 ++++++++++++++++++++++++++ elk/amavis.grok | 11 +++++++++++ target/filebeat.yml.tmpl | 13 +++++++++++++ target/start-mailserver.sh | 12 ++++++++++++ 8 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 docker-compose.elk.yml.dist create mode 100644 elk/02-beats-input.conf create mode 100644 elk/10-syslog.conf create mode 100644 elk/Dockerfile create mode 100644 elk/amavis.grok create mode 100644 target/filebeat.yml.tmpl diff --git a/Dockerfile b/Dockerfile index 37a24ecd..77346926 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -q --fix-missing && \ && \ curl -sk http://neuro.debian.net/lists/trusty.de-m.libre > /etc/apt/sources.list.d/neurodebian.sources.list && \ apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ - apt-get update -q --fix-missing && apt-get -y upgrade fail2ban && \ + curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ + echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ + apt-get update -q --fix-missing && apt-get -y upgrade fail2ban filebeat && \ apt-get autoclean && rm -rf /var/lib/apt/lists/* && \ rm -rf /usr/share/locale/* && rm -rf /usr/share/man/* && rm -rf /usr/share/doc/* @@ -112,3 +114,8 @@ RUN chmod +x /usr/local/bin/* EXPOSE 25 587 143 993 110 995 4190 CMD /usr/local/bin/start-mailserver.sh + + +ADD target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl + + diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist new file mode 100644 index 00000000..75ffa10e --- /dev/null +++ b/docker-compose.elk.yml.dist @@ -0,0 +1,34 @@ +version: '2' + +services: + mail: + image: tvial/docker-mailserver:v2 + hostname: mail + domainname: domain.com + container_name: mail + links: + - elk + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + volumes: + - maildata:/var/mail + - ./config/:/tmp/docker-mailserver/ + environment: + - ENABLE_FAIL2BAN=1 + - ENABLE_ELK_FORWARDER=1 + cap_add: + - NET_ADMIN + elk: + build: elk + ports: + - "5601:5601" + - "9200:9200" + - "5044:5044" + - "5000:5000" + +volumes: + maildata: + driver: local diff --git a/elk/02-beats-input.conf b/elk/02-beats-input.conf new file mode 100644 index 00000000..a00d3f5b --- /dev/null +++ b/elk/02-beats-input.conf @@ -0,0 +1,6 @@ +input { + beats { + port => 5044 + ssl => false + } +} diff --git a/elk/10-syslog.conf b/elk/10-syslog.conf new file mode 100644 index 00000000..ae03326a --- /dev/null +++ b/elk/10-syslog.conf @@ -0,0 +1,14 @@ +filter { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{host}" ] + add_field => [ "program", "%{syslog_program}" ] + } + syslog_pri { } + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } +} + + diff --git a/elk/Dockerfile b/elk/Dockerfile new file mode 100644 index 00000000..dc62c270 --- /dev/null +++ b/elk/Dockerfile @@ -0,0 +1,26 @@ +FROM sebp/elk + +RUN mkdir /etc/logstash/patterns.d +#postfix grok and filter +RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok +RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf +# custom amavis grok and filter +ADD amavis.grok /etc/logstash/patterns.d +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-amavis.conf > /etc/logstash/conf.d/16-filter-amavis.conf +# dovecot grok and filter +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf +# FIXME: may be a cron job? +RUN mkdir -p /usr/share/GeoIP && \ + curl -L http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat + +WORKDIR ${LOGSTASH_HOME} +RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip + +# override beats input +ADD 02-beats-input.conf /etc/logstash/conf.d/ +# override syslog +ADD 10-syslog.conf /etc/logstash/conf.d/ + + + diff --git a/elk/amavis.grok b/elk/amavis.grok new file mode 100644 index 00000000..4bc74859 --- /dev/null +++ b/elk/amavis.grok @@ -0,0 +1,11 @@ +MAVIS_MESSAGEID Message-ID: <%{DATA:amavis_message-id}> +AMAVIS_SIZE size: %{POSINT:amavis_size} +AMAVIS_TESTS Tests: \[%{DATA:amavis_tests}\] +AMAVIS_FROM From: %{DATA:amavis_header_from} +AMAVIS_HITS Hits: %{NUMBER:amavis_hits} +AMAVIS_QUARANTINE quarantine: %{NOTSPACE:amavis_quarantine} +AMAVIS_SUBJECT Subject: "%{DATA:amavis_subject}" +AMAVIS_KV ((%{AMAVIS_MESSAGEID}|%{AMAVIS_SIZE}|%{AMAVIS_TESTS}|%{AMAVIS_FROM}|%{AMAVIS_HITS}|%{AMAVIS_QUARANTINE}|%{AMAVIS_SUBJECT}|%{DATA}), )* + +AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms + diff --git a/target/filebeat.yml.tmpl b/target/filebeat.yml.tmpl new file mode 100644 index 00000000..d2d84ed4 --- /dev/null +++ b/target/filebeat.yml.tmpl @@ -0,0 +1,13 @@ +output: + logstash: + enabled: true + hosts: + - $ELK_HOST:$ELK_PORT + +filebeat: + prospectors: + - + paths: + - /var/log/mail/mail.log + document_type: syslog + diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index 81611b72..5d3aeae0 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -354,10 +354,22 @@ if [ "$ONE_DIR" = 1 -a -d $statedir ]; then fi done fi +if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then +ELK_PORT=${ELK_PORT:="5044"} +ELK_HOST=${ELK_HOST:="elk"} +echo "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)" +cat /etc/filebeat/filebeat.yml.tmpl \ + | sed "s@\$ELK_HOST@$ELK_HOST@g" \ + | sed "s@\$ELK_PORT@$ELK_PORT@g" \ + > /etc/filebeat/filebeat.yml +fi echo "Starting daemons" cron /etc/init.d/rsyslog start +if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then +/etc/init.d/filebeat start +fi # Enable Managesieve service by setting the symlink # to the configuration file Dovecot will actually find