add ELK support (#331)

* add support to forward logs to ELK stack.
* from docker elk customize image with
* https://github.com/whyscream/postfix-grok-patterns
* custom imput
* override syslog filter.
* fix typo.
* Explicit forwarder vars and messages.
* add amavis grok
* add dovecot grok
* add geoip db
* add logstash geoip plugin
* add custom amavis grok from @tomav.
* switch to filebeats input
* refactor syslog filter
* add filebeat
* add template config
* replace rsyslog with filebeat.

Dockerfile

@@ -39,7 +39,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -q --fix-missing && \
     && \
   curl -sk http://neuro.debian.net/lists/trusty.de-m.libre > /etc/apt/sources.list.d/neurodebian.sources.list && \
   apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \
-  apt-get update -q --fix-missing && apt-get -y upgrade fail2ban && \
+  curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \
+  echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \
+  apt-get update -q --fix-missing && apt-get -y upgrade fail2ban filebeat && \
   apt-get autoclean && rm -rf /var/lib/apt/lists/* && \
   rm -rf /usr/share/locale/* && rm -rf /usr/share/man/* && rm -rf /usr/share/doc/*
 
@@ -112,3 +114,8 @@ RUN chmod +x /usr/local/bin/*
 EXPOSE 25 587 143 993 110 995 4190
 
 CMD /usr/local/bin/start-mailserver.sh
+
+
+ADD target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl
+
+

docker-compose.elk.yml.dist

@@ -0,0 +1,34 @@
+version: '2'
+
+services:
+  mail:
+    image: tvial/docker-mailserver:v2
+    hostname: mail
+    domainname: domain.com
+    container_name: mail
+    links:
+    - elk
+    ports:
+    - "25:25"
+    - "143:143"
+    - "587:587"
+    - "993:993"
+    volumes:
+    - maildata:/var/mail
+    - ./config/:/tmp/docker-mailserver/
+    environment:
+    - ENABLE_FAIL2BAN=1
+    - ENABLE_ELK_FORWARDER=1
+    cap_add:
+    - NET_ADMIN
+  elk:
+     build: elk
+     ports:
+     - "5601:5601"
+     - "9200:9200"
+     - "5044:5044"
+     - "5000:5000"
+
+volumes:
+  maildata:
+    driver: local

elk/02-beats-input.conf

@@ -0,0 +1,6 @@
+input {
+  beats {
+    port => 5044
+    ssl => false
+  }
+}

elk/10-syslog.conf

@@ -0,0 +1,14 @@
+filter {
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+      add_field => [ "program", "%{syslog_program}" ]
+    }
+    syslog_pri { }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+    }
+}
+
+

elk/Dockerfile

@@ -0,0 +1,26 @@
+FROM sebp/elk
+
+RUN mkdir /etc/logstash/patterns.d
+#postfix grok and filter
+RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok
+RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf
+# custom amavis grok and filter
+ADD amavis.grok  /etc/logstash/patterns.d
+RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-amavis.conf > /etc/logstash/conf.d/16-filter-amavis.conf
+# dovecot grok and filter
+RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok
+RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf
+# FIXME: may be a cron job? 
+RUN mkdir  -p /usr/share/GeoIP && \
+ curl -L http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat 
+
+WORKDIR ${LOGSTASH_HOME}
+RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip
+
+# override beats input 
+ADD 02-beats-input.conf /etc/logstash/conf.d/
+# override syslog
+ADD 10-syslog.conf /etc/logstash/conf.d/
+
+
+

elk/amavis.grok

@@ -0,0 +1,11 @@
+MAVIS_MESSAGEID Message-ID: <%{DATA:amavis_message-id}>
+AMAVIS_SIZE size: %{POSINT:amavis_size}
+AMAVIS_TESTS Tests: \[%{DATA:amavis_tests}\]
+AMAVIS_FROM From: %{DATA:amavis_header_from}
+AMAVIS_HITS Hits: %{NUMBER:amavis_hits}
+AMAVIS_QUARANTINE quarantine: %{NOTSPACE:amavis_quarantine}
+AMAVIS_SUBJECT Subject: "%{DATA:amavis_subject}"
+AMAVIS_KV ((%{AMAVIS_MESSAGEID}|%{AMAVIS_SIZE}|%{AMAVIS_TESTS}|%{AMAVIS_FROM}|%{AMAVIS_HITS}|%{AMAVIS_QUARANTINE}|%{AMAVIS_SUBJECT}|%{DATA}), )*
+
+AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms
+

target/filebeat.yml.tmpl

@@ -0,0 +1,13 @@
+output:
+  logstash:
+    enabled: true
+    hosts:
+      - $ELK_HOST:$ELK_PORT
+
+filebeat:
+  prospectors:
+    -
+      paths:
+        - /var/log/mail/mail.log
+      document_type: syslog
+

target/start-mailserver.sh

@@ -354,10 +354,22 @@ if [ "$ONE_DIR" = 1 -a -d $statedir ]; then
     fi
   done
 fi
+if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
+ELK_PORT=${ELK_PORT:="5044"}
+ELK_HOST=${ELK_HOST:="elk"}
+echo "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)"
+cat /etc/filebeat/filebeat.yml.tmpl \
+	| sed "s@\$ELK_HOST@$ELK_HOST@g" \
+	| sed "s@\$ELK_PORT@$ELK_PORT@g" \
+	 > /etc/filebeat/filebeat.yml
+fi
 
 echo "Starting daemons"
 cron
 /etc/init.d/rsyslog start
+if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
+/etc/init.d/filebeat start
+fi
 
 # Enable Managesieve service by setting the symlink
 # to the configuration file Dovecot will actually find