From b6fba0c03feeb39c7db36c05da70280f1ca270fb Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:43:55 +0200 Subject: [PATCH 1/5] filer: make k8s service ClusterIP type easier to patch/expose ( probably LoadBalancer will be better, but that also works) --- k8s/seaweedfs/templates/filer-service-client.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/k8s/seaweedfs/templates/filer-service-client.yaml b/k8s/seaweedfs/templates/filer-service-client.yaml index 65568418f..f509086e3 100644 --- a/k8s/seaweedfs/templates/filer-service-client.yaml +++ b/k8s/seaweedfs/templates/filer-service-client.yaml @@ -10,7 +10,6 @@ metadata: monitoring: "true" {{- end }} spec: - clusterIP: None ports: - name: "swfs-filer" port: {{ .Values.filer.port }} @@ -28,4 +27,4 @@ spec: {{- end }} selector: app: {{ template "seaweedfs.name" . }} - component: filer \ No newline at end of file + component: filer From 2646f5a7847ffc2117861eaaab749a63f5c471e5 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:46:21 +0200 Subject: [PATCH 2/5] cronjob: fix and update enable/disable fix.replication & support for CollectionPrefix filtering fix toleration and nodeSelector helm templating issues --- k8s/seaweedfs/templates/cronjob.yaml | 15 +++++++++------ k8s/seaweedfs/values.yaml | 16 ++++++++++------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/k8s/seaweedfs/templates/cronjob.yaml b/k8s/seaweedfs/templates/cronjob.yaml index 6f4ed8c70..4caf4bad1 100644 --- a/k8s/seaweedfs/templates/cronjob.yaml +++ b/k8s/seaweedfs/templates/cronjob.yaml @@ -15,13 +15,13 @@ spec: backoffLimit: 2 template: spec: - {{- with .Values.cronjob.nodeSelector }} + {{- if .Values.cronjob.nodeSelector }} nodeSelector: - {{- toYaml . | nindent 12 }} + {{ tpl .Values.cronjob.nodeSelector . | indent 12 | trim }} {{- end }} - {{- with .Values.cronjob.tolerations }} + {{- if .Values.cronjob.tolerations }} tolerations: - {{- toYaml . | nindent 12 }} + {{ tpl .Values.cronjob.tolerations . | nindent 12 | trim }} {{- end }} restartPolicy: OnFailure containers: @@ -36,10 +36,13 @@ spec: - | set -ex echo -e "lock\n\ - volume.balance -force\ + volume.balance -force \ {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\ {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\ - volume.fix.replication\nunlock\n" | \ + {{- if .Values.cronjob.enableFixReplication }} + volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\ + {{- end }} + unlock\n" | \ /usr/bin/weed shell \ {{- if .Values.cronjob.master }} -master {{ .Values.cronjob.master }} \ diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 2b33a6149..301c9f236 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -359,17 +359,21 @@ s3: storageClass: "" cronjob: - enabled: false + enabled: true + master: "seaweedfs-master:9333" + filer: "seaweedfs-filer-client:8888" + tolerations: "" + nodeSelector: | + sw-backend: "true" + replication: + enable: true + collectionPattern: "" schedule: "*/7 * * * *" resources: null # balance all volumes among volume servers # ALL|EACH_COLLECTION| collection: "" - master: "" - filer: "" - tolerations: "" - nodeSelector: | - sw-backend: "true" + certificates: commonName: "SeaweedFS CA" From 91a3314d36445c84d337963147d0ed30bde21599 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:55:58 +0200 Subject: [PATCH 3/5] filer/s3: enable/disable s3 and peers enable/disable s3 pods launch s3 in filer pod, to reduce network/latency between pods when using s3 disable/enable the peers option (with external backend DB is not needed) added option to create s3 admin+read users --- .../templates/filer-statefulset.yaml | 29 +++++++++++++++++++ k8s/seaweedfs/templates/s3-deployment.yaml | 10 +++++++ k8s/seaweedfs/templates/s3-service.yaml | 10 +++---- k8s/seaweedfs/values.yaml | 18 +++++++++++- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/k8s/seaweedfs/templates/filer-statefulset.yaml b/k8s/seaweedfs/templates/filer-statefulset.yaml index d284e9992..fc1253479 100644 --- a/k8s/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/seaweedfs/templates/filer-statefulset.yaml @@ -133,14 +133,36 @@ spec: -encryptVolumeData \ {{- end }} -ip=${POD_IP} \ + {{- if .Values.filer.enable_peers }} {{- if gt (.Values.filer.replicas | int) 1 }} -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' ) \ {{- end }} + {{- end }} + {{- if .Values.filer.s3.enabled }} + -s3 \ + -s3.port={{ .Values.filer.s3.port }} \ + {{- if .Values.filer.s3.domainName }} + -s3.domainName={{ .Values.filer.s3.domainName }} \ + {{- end }} + {{- if .Values.global.enableSecurity }} + -s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \ + -s3.key.file=/usr/local/share/ca-certificates/client/tls.key \ + {{- end }} + {{- if .Values.filer.s3.allowEmptyFolder }} + -s3.allowEmptyFolder={{ .Values.filer.s3.allowEmptyFolder }} \ + {{- end }} + {{- if .Values.filer.s3.enableAuth }} + -s3.config=/etc/sw/seaweedfs_s3_config \ + {{- end }} + {{- end }} -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }} volumeMounts: - name: seaweedfs-filer-log-volume mountPath: "/logs/" + - mountPath: /etc/sw + name: config-users + readOnly: true {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -198,6 +220,13 @@ spec: hostPath: path: /storage/logs/seaweedfs/filer type: DirectoryOrCreate + - name: db-schema-config-volume + configMap: + name: seaweedfs-db-init-config + - name: config-users + secret: + defaultMode: 420 + secretName: seaweedfs-s3-secret {{- if .Values.global.enableSecurity }} - name: security-config configMap: diff --git a/k8s/seaweedfs/templates/s3-deployment.yaml b/k8s/seaweedfs/templates/s3-deployment.yaml index ed01758b8..b513e937b 100644 --- a/k8s/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/seaweedfs/templates/s3-deployment.yaml @@ -90,10 +90,16 @@ spec: {{- if .Values.s3.allowEmptyFolder }} -allowEmptyFolder={{ .Values.s3.allowEmptyFolder }} \ {{- end }} + {{- if .Values.s3.enableAuth }} + -config=/etc/sw/seaweedfs_s3_config \ + {{- end }} -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }} volumeMounts: - name: logs mountPath: "/logs/" + - mountPath: /etc/sw + name: config-users + readOnly: true {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -144,6 +150,10 @@ spec: {{ tpl .Values.s3.resources . | nindent 12 | trim }} {{- end }} volumes: + - name: config-users + secret: + defaultMode: 420 + secretName: seaweedfs-s3-secret {{- if eq .Values.s3.logs.type "hostPath" }} - name: logs hostPath: diff --git a/k8s/seaweedfs/templates/s3-service.yaml b/k8s/seaweedfs/templates/s3-service.yaml index 4a68c7976..122b33298 100644 --- a/k8s/seaweedfs/templates/s3-service.yaml +++ b/k8s/seaweedfs/templates/s3-service.yaml @@ -9,15 +9,15 @@ metadata: spec: ports: - name: "swfs-s3" - port: {{ .Values.s3.port }} - targetPort: {{ .Values.s3.port }} + port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }} + targetPort: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }} protocol: TCP -{{- if .Values.s3.metricsPort }} - - name: "swfs-s3-metrics" +{{- if and .Values.s3.enabled .Values.s3.metricsPort }} + - name: "metrics" port: {{ .Values.s3.metricsPort }} targetPort: {{ .Values.s3.metricsPort }} protocol: TCP {{- end }} selector: app: {{ template "seaweedfs.name" . }} - component: s3 \ No newline at end of file + component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }} diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 301c9f236..43783d3b7 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -229,6 +229,8 @@ filer: maxMB: null # encrypt data on volume servers encryptVolumeData: false + # enable peers sync metadata, for leveldb (localdb for filer but with sync across) + enable_peers: false # Whether proxy or redirect to volume server during file GET request redirectOnRead: false @@ -311,8 +313,19 @@ filer: # directories under this folder will be automatically creating a separate bucket WEED_FILER_BUCKETS_FOLDER: "/buckets" + s3: + enabled: true + port: 8333 + #allow empty folders + allowEmptyFolder: false + # Suffix of the host name, {bucket}.{domainName} + domainName: "" + # enable user & permission to s3 (need to inject to all services) + enableAuth: false + skipAuthSecretCreation: false + s3: - enabled: true + enabled: false repository: null imageName: null imageTag: null @@ -323,6 +336,9 @@ s3: loggingOverrideLevel: null #allow empty folders allowEmptyFolder: true + # enable user & permission to s3 (need to inject to all services) + enableAuth: false + skipAuthSecretCreation: false # Suffix of the host name, {bucket}.{domainName} domainName: "" From 259c66d978f7d296325fcc20ab7eceb127c377c8 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 09:57:09 +0200 Subject: [PATCH 4/5] volume: set minFreeSpacePercent to 7% when k8s node have less than 5% free space, the k8s will start killing pods... --- k8s/seaweedfs/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 43783d3b7..273ab21ad 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -136,7 +136,7 @@ volume: # limit file size to avoid out of memory, default 256mb fileSizeLimitMB: null # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly - minFreeSpacePercent: 1 + minFreeSpacePercent: 7 # limit background compaction or copying speed in mega bytes per second From b373c1de93ab172ca8f0c3b15329ccea74f5b3c9 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Tue, 9 Feb 2021 10:07:25 +0200 Subject: [PATCH 5/5] remove collectionPattern if is empty string / null --- k8s/seaweedfs/templates/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/seaweedfs/templates/cronjob.yaml b/k8s/seaweedfs/templates/cronjob.yaml index 4caf4bad1..c7dcd52b1 100644 --- a/k8s/seaweedfs/templates/cronjob.yaml +++ b/k8s/seaweedfs/templates/cronjob.yaml @@ -40,7 +40,7 @@ spec: {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\ {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\ {{- if .Values.cronjob.enableFixReplication }} - volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\ + volume.fix.replication {{ if .Values.cronjob.collectionPattern }} -collectionPattern={{ .Values.cronjob.collectionPattern }} {{ end }} \n\ {{- end }} unlock\n" | \ /usr/bin/weed shell \