getting rid of ELK & updating Compose (#1614)

* removed elk diretory * removed filebeat / elk compose.yml.dist; adjusted other compose files * final README update for env variables * updated compose files
2024-01-19 02:48:50 +00:00 · 2020-09-21 13:54:31 +02:00 · 2020-09-21 13:54:31 +02:00 · 9b5d4d307c
parent fa84895ab2
commit 9b5d4d307c
13 changed files with 33 additions and 241 deletions
--- a/README.md
+++ b/README.md
@ -179,9 +179,9 @@ version: '3.8'
 services:
  mail:
    image: tvial/docker-mailserver:latest
-    hostname: mail
-    domainname: domain.com
-    container_name: mail
+    hostname: mail                         # ${HOSTNAME}
+    domainname: domain.com                 # ${DOMAINNAME}
+    container_name: mail                   # ${CONTAINER_NAME}
    ports:
      - "25:25"
      - "143:143"
@ -203,14 +203,12 @@ services:
    cap_add:
      - NET_ADMIN
      - SYS_PTRACE
+    restart: always

 volumes:
  maildata:
-    driver: local
  mailstate:
-    driver: local
  maillogs:
-    driver: local
 ```

 #### LDAP setup
@ -221,9 +219,9 @@ version: '3.8'
 services:
  mail:
    image: tvial/docker-mailserver:latest
-    hostname: mail
-    domainname: domain.com
-    container_name: mail
+    hostname: mail                         # ${HOSTNAME}
+    domainname: domain.com                 # ${DOMAINNAME}
+    container_name: mail                   # ${CONTAINER_NAME}
    ports:
      - "25:25"
      - "143:143"
@ -265,19 +263,21 @@ services:
    cap_add:
      - NET_ADMIN
      - SYS_PTRACE
+    restart: always

 volumes:
  maildata:
-    driver: local
  mailstate:
-    driver: local
  maillogs:
-    driver: local
 ```

 ## Environment variables

-If an option doesn't work as documented here, check if you are running the latest image! Value in **bold** is the default value.
+If an option doesn't work as documented here, check if you are running the latest image! Values in **bold** are the default values.
+
+### Reminder
+
+Please note: Variables in `.env` are expanded in the `docker-compose.yml` file **only** and **not** in the container. The file `env-mailserver` serves this case where environment variables are used in the container.

 ### Assignments

--- a/config/filebeat.docker.yml
+++ b/config/filebeat.docker.yml
@ -1,16 +0,0 @@
-filebeat.config:
-  modules:
-    path: ${path.config}/modules.d/*.yml
-    reload.enabled: false
-
-filebeat.autodiscover:
-  providers:
-    - type: docker
-      hints.enabled: true
-      hints.default_config.enabled: false
-
-processors:
- add_cloud_metadata: ~
-
-output.logstash:
-    hosts: ["127.0.0.1:5044"]
--- a/docker-compose.elk.yml.dist
+++ b/docker-compose.elk.yml.dist
@ -1,58 +0,0 @@
-version: '2'
-
-services:
-  mail:
-    image: tvial/docker-mailserver:latest
-    hostname: ${HOSTNAME}
-    domainname: ${DOMAINNAME}
-    container_name: ${CONTAINER_NAME}
-    links:
-    - elk
-    labels:
-    - "co.elastic.logs/enabled=true"
-    - "co.elastic.logs/module=system"
-    - "co.elastic.logs/fileset.stdout=syslog"
-    ports:
-    - "25:25"
-    - "143:143"
-    - "587:587"
-    - "993:993"
-    volumes:
-    - maildata:/var/mail
-    - maillogs:/var/log/mail
-    - ./config/:/tmp/docker-mailserver/
-    env_file:
-    - .env
-    - env-mailserver
-    cap_add:
-    - NET_ADMIN
-    - SYS_PTRACE
-    restart: always
-  filebeat:
-    image: docker.elastic.co/beats/filebeat:7.6.1
-    user: root
-    volumes:
-    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
-    - /var/run/docker.sock:/var/run/docker.sock:ro
-    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
-    command: ["filebeat", "-e", "--strict.perms=false"]
-    restart: always
-  elk:
-     build:
-       context: elk
-       args:
-       - MAXMIND_LICENSE
-     ports:
-     - "5601:5601"
-     - "9200:9200"
-     - "5044:5044"
-     - "5000:5000"
-     env_file:
-     - elk/.env
-     restart: always
-
-volumes:
-  maildata:
-    driver: local
-  maillogs:
-    driver: local
--- a/docker-compose.filebeat.yml.dist
+++ b/docker-compose.filebeat.yml.dist
@ -1,42 +0,0 @@
-version: '2'
-services:
-  mail:
-    image: tvial/docker-mailserver:latest
-    hostname: ${HOSTNAME}
-    domainname: ${DOMAINNAME}
-    container_name: ${CONTAINER_NAME}
-    ports:
-    - "25:25"
-    - "143:143"
-    - "587:587"
-    - "993:993"
-    labels:
-    - "co.elastic.logs/enabled=true"
-    - "co.elastic.logs/module=system"
-    - "co.elastic.logs/fileset.stdout=syslog"
-    volumes:
-    - maildata:/var/mail
-    - mailstate:/var/mail-state
-    - maillogs:/var/log/mail
-    - ./config/:/tmp/docker-mailserver/
-    env_file:
-    - .env
-    - env-mailserver
-    cap_add:
-    - NET_ADMIN
-    - SYS_PTRACE
-    restart: always
-  filebeat:
-    image: docker.elastic.co/beats/filebeat:7.6.1
-    user: root
-    volumes:
-    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
-    - /var/run/docker.sock:/var/run/docker.sock:ro
-    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
-    command: ["filebeat", "-e", "--strict.perms=false"]
-    restart: always
-volumes:
-  maildata:
-    driver: local
-  maillogs:
-    driver: local
--- a/docker-compose.yml.dist
+++ b/docker-compose.yml.dist
@ -1,4 +1,5 @@
-version: '2'
+version: '3.7'
+
 services:
  mail:
    image: tvial/docker-mailserver:latest
@ -6,26 +7,24 @@ services:
    domainname: ${DOMAINNAME}
    container_name: ${CONTAINER_NAME}
    ports:
-    - "25:25"
-    - "143:143"
-    - "587:587"
-    - "993:993"
+      - "25:25"
+      - "143:143"
+      - "587:587"
+      - "993:993"
    volumes:
-    - maildata:/var/mail
-    - mailstate:/var/mail-state
-    - maillogs:/var/log/mail
-    - ./config/:/tmp/docker-mailserver/
+      - maildata:/var/mail
+      - mailstate:/var/mail-state
+      - maillogs:/var/log/mail
+      - ./config/:/tmp/docker-mailserver/
    env_file:
-    - .env
-    - env-mailserver
+      - .env
+      - env-mailserver
    cap_add:
-    - NET_ADMIN
-    - SYS_PTRACE
+      - NET_ADMIN
+      - SYS_PTRACE
    restart: always
+
 volumes:
  maildata:
-    driver: local
  mailstate:
-    driver: local
  maillogs:
-    driver: local
--- a/elk/.env.dist
+++ b/elk/.env.dist
@ -1 +0,0 @@
-MAXMIND_LICENSE=
--- a/elk/02-beats-input.conf
+++ b/elk/02-beats-input.conf
@ -1,6 +0,0 @@
-input {
-  beats {
-    port => 5044
-    ssl => false
-  }
-}
--- a/elk/10-syslog.conf
+++ b/elk/10-syslog.conf
@ -1,15 +0,0 @@
-filter {
-    grok {
-      overwrite => [ "message" ]
-      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:message}" }
-      add_field => [ "received_at", "%{@timestamp}" ]
-      add_field => [ "received_from", "%{host}" ]
-      add_field => [ "program", "%{syslog_program}" ]
-    }
-    syslog_pri { }
-    date {
-      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
-    }
-}
-
-
--- a/elk/16-amavis.conf
+++ b/elk/16-amavis.conf
@ -1,23 +0,0 @@
-filter {
-    # grok log lines by program name
-    if [program] == 'amavis' {
-        grok {
-            patterns_dir   => "/etc/logstash/patterns.d"
-            match          => [ "message", "%{AMAVIS}" ]
-            tag_on_failure => [ "_grok_amavis_nomatch" ]
-            add_tag        => [ "_grok_amavis_success" ]
-        }
-    }
-
-    # Do some data type conversions
-    mutate {
-        convert => [
-            # list of integer fields
-            "amavis_size", "integer",
-            "amavis_duration", "integer",
-
-            # list of float fields
-            "amavis_hits", "float"
-        ]
-    }
-}
--- a/elk/Dockerfile
+++ b/elk/Dockerfile
@ -1,30 +0,0 @@
-FROM sebp/elk:761
-
-RUN mkdir /etc/logstash/patterns.d
-#postfix grok and filter
-RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok
-RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf
-# custom amavis grok and filter
-COPY amavis.grok  /etc/logstash/patterns.d
-COPY 16-amavis.conf /etc/logstash/conf.d
-# dovecot grok and filter
-RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok
-RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf
-# FIXME: may be a cron job?
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-ARG MAXMIND_LICENSE
-RUN mkdir  -p /usr/share/GeoIP && \
-curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \
-| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat
-
-WORKDIR ${LOGSTASH_HOME}
-RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip
-
-# override beats input
-COPY 02-beats-input.conf /etc/logstash/conf.d/
-# override syslog
-COPY 10-syslog.conf /etc/logstash/conf.d/
-
-# avoid Bootstrap Checks failure on production
-RUN /bin/grep -q  -F 'transport.host' /etc/elasticsearch/elasticsearch.yml || echo "transport.host: 127.0.0.1" >> /etc/elasticsearch/elasticsearch.yml
--- a/elk/amavis.grok
+++ b/elk/amavis.grok
@ -1 +0,0 @@
-AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms
--- a/elk/docker-compose.yml.dist
+++ b/elk/docker-compose.yml.dist
@ -1,16 +0,0 @@
-version: '2'
-
-services:
-  elk:
-     build:
-       context: .
-       args:
-       - MAXMIND_LICENSE
-     ports:
-     - "5601:5601"
-     - "9200:9200"
-     - "5044:5044"
-     - "5000:5000"
-     env_file:
-     - .env
-     restart: always
--- a/setup.sh
+++ b/setup.sh
@ -4,7 +4,7 @@
 # included in the docker-mailserver

 set -euEo pipefail
-trap '_report_err ${_:-"SOURCE UNKNOWN"} ${LINENO} ${?}' ERR
+trap '_report_err ${_} ${LINENO} ${?}' ERR

 function _report_err()
 {
@ -321,13 +321,14 @@ function _main()
            _docker_container /bin/bash -c "${@}"
          fi
          ;;
-        *        ) _usage ; exit 1 ;;
+        *        ) _usage ; _unset_vars ; exit 1 ;;
      esac
      ;;

-    *            ) _usage ; exit 1 ;;
+    *            ) _usage ; _unset_vars ; exit 1 ;;
  esac
+
+  _unset_vars
 }

 _main "${@}"
-_unset_vars
				`@ -1 +0,0 @@`
				AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms