Merge pull request #1434 from gmasse/filebeat-in-container

Filebeat in its own container

Dockerfile

@@ -119,17 +119,6 @@ RUN \
   rm -f /etc/postsrsd.secret && \
   rm -f /etc/cron.daily/00logwatch
 
-# install filebeat for logging
-# SKIP and run in an external container instead
-#RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \
-#  echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \
-#  apt-get update -q --fix-missing && \
-#  apt-get -y install --no-install-recommends \
-#    filebeat \
-#  && apt-get clean \
-#  && rm -rf /var/lib/apt/lists/*
-#COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl
-
 RUN echo "0 */6 * * * clamav /usr/bin/freshclam --quiet" > /etc/cron.d/clamav-freshclam && \
   chmod 644 /etc/clamav/freshclam.conf && \
   freshclam && \

README.md

@@ -109,7 +109,7 @@ If you got any problems with SPF and/or forwarding mails, give [SRS](https://git
 
 Your config folder will be mounted in `/tmp/docker-mailserver/`. To understand how things work on boot, please have a look at [start-mailserver.sh](https://github.com/tomav/docker-mailserver/blob/master/target/start-mailserver.sh)
 
-`restart: always` ensures that the mail server container (and ELK container when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit.
+`restart: always` ensures that the mail server container (and Filebeat/ELK containers when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit.
 
 #### Exposed ports
 * 25 receiving email from other mailservers

config/filebeat.docker.yml

@@ -0,0 +1,16 @@
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      hints.default_config.enabled: false
+
+processors:
+- add_cloud_metadata: ~
+
+output.logstash:
+    hosts: ["127.0.0.1:5044"]

docker-compose.elk.yml.dist

@@ -8,6 +8,10 @@ services:
     container_name: ${CONTAINER_NAME}
     links:
     - elk
+    labels:
+    - "co.elastic.logs/enabled=true"
+    - "co.elastic.logs/module=system"
+    - "co.elastic.logs/fileset.stdout=syslog"
     ports:
     - "25:25"
     - "143:143"
@@ -24,6 +28,15 @@ services:
     - NET_ADMIN
     - SYS_PTRACE
     restart: always
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.6.1
+    user: root
+    volumes:
+    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
+    command: ["filebeat", "-e", "--strict.perms=false"]
+    restart: always
   elk:
      build: elk
      ports:

docker-compose.filebeat.yml.dist

@@ -0,0 +1,42 @@
+version: '2'
+services:
+  mail:
+    image: tvial/docker-mailserver:latest
+    hostname: ${HOSTNAME}
+    domainname: ${DOMAINNAME}
+    container_name: ${CONTAINER_NAME}
+    ports:
+    - "25:25"
+    - "143:143"
+    - "587:587"
+    - "993:993"
+    labels:
+    - "co.elastic.logs/enabled=true"
+    - "co.elastic.logs/module=system"
+    - "co.elastic.logs/fileset.stdout=syslog"
+    volumes:
+    - maildata:/var/mail
+    - mailstate:/var/mail-state
+    - maillogs:/var/log/mail
+    - ./config/:/tmp/docker-mailserver/
+    env_file:
+    - .env
+    - env-mailserver
+    cap_add:
+    - NET_ADMIN
+    - SYS_PTRACE
+    restart: always
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.6.1
+    user: root
+    volumes:
+    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
+    command: ["filebeat", "-e", "--strict.perms=false"]
+    restart: always
+volumes:
+  maildata:
+    driver: local
+  maillogs:
+    driver: local

target/start-mailserver.sh

@@ -95,10 +95,6 @@ function register_functions() {
 	_register_setup_function "_setup_default_vars"
 	_register_setup_function "_setup_file_permissions"
 
-	if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
-		_register_setup_function "_setup_elk_forwarder"
-	fi
-
 	if [ "$SMTP_ONLY" != 1 ]; then
 		_register_setup_function "_setup_dovecot"
                 _register_setup_function "_setup_dovecot_dhparam"
@@ -208,10 +204,6 @@ function register_functions() {
 	_register_start_daemon "_start_daemons_cron"
 	_register_start_daemon "_start_daemons_rsyslog"
 
-	if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
-		_register_start_daemon "_start_daemons_filebeat"
-	fi
-
 	if [ "$SMTP_ONLY" != 1 ]; then
 		_register_start_daemon "_start_daemons_dovecot"
 	fi
@@ -1452,18 +1444,6 @@ function _setup_security_stack() {
 	fi
 }
 
-function _setup_elk_forwarder() {
-	notify 'task' 'Setting up Elk forwarder'
-
-	ELK_PORT=${ELK_PORT:="5044"}
-	ELK_HOST=${ELK_HOST:="elk"}
-	notify 'inf' "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)"
-	cat /etc/filebeat/filebeat.yml.tmpl \
-		| sed "s@\$ELK_HOST@$ELK_HOST@g" \
-		| sed "s@\$ELK_PORT@$ELK_PORT@g" \
-		> /etc/filebeat/filebeat.yml
-}
-
 function _setup_logrotate() {
 	notify 'inf' "Setting up logrotate"
 
@@ -1750,11 +1730,6 @@ function _start_daemons_dovecot() {
 	#fi
 }
 
-function _start_daemons_filebeat() {
-	notify 'task' 'Starting filebeat' 'n'
-    supervisorctl start filebeat
-}
-
 function _start_daemons_fetchmail() {
 	notify 'task' 'Starting fetchmail' 'n'
 	/usr/local/bin/setup-fetchmail