diff --git a/k8s/seaweedfs/templates/filer-statefulset.yaml b/k8s/seaweedfs/templates/filer-statefulset.yaml index 43da74c43..d1dd851fe 100644 --- a/k8s/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/seaweedfs/templates/filer-statefulset.yaml @@ -99,6 +99,9 @@ spec: {{- end }} filer \ -port={{ .Values.filer.port }} \ + {{- if .Values.filer.redirectOnRead }} + -redirectOnRead \ + {{- end }} {{- if .Values.filer.disableHttp }} -disableHttp \ {{- end }} @@ -106,7 +109,24 @@ spec: -disableDirListing \ {{- end }} -dirListLimit={{ .Values.filer.dirListLimit }} \ + {{- if .Values.global.enableReplication }} + -defaultReplicaPlacement={{ .Values.global.replicationPlacment }} \ + {{- else }} + -defaultReplicaPlacement={{ .Values.filer.defaultReplicaPlacement }} \ + {{- end }} + {{- if .Values.filer.disableDirListing }} + -disableDirListing \ + {{- end }} + {{- if .Values.filer.maxMB }} + -maxMB={{ .Values.filer.maxMB }} \ + {{- end }} + {{- if .Values.filer.encryptVolumeData }} + -encryptVolumeData \ + {{- end }} -ip=${POD_IP} \ + {{- if gt (.Values.filer.replicas | int) 1 }} + -peers={{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }} + {{- end }} -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }} volumeMounts: @@ -149,6 +169,7 @@ spec: periodSeconds: 15 successThreshold: 1 failureThreshold: 100 + timeoutSeconds: 3 livenessProbe: httpGet: path: / @@ -158,6 +179,7 @@ spec: periodSeconds: 30 successThreshold: 1 failureThreshold: 5 + timeoutSeconds: 3 {{- if .Values.filer.resources }} resources: {{ tpl .Values.filer.resources . | nindent 12 | trim }} diff --git a/k8s/seaweedfs/templates/master-statefulset.yaml b/k8s/seaweedfs/templates/master-statefulset.yaml index 87050534f..fe90f3d81 100644 --- a/k8s/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/seaweedfs/templates/master-statefulset.yaml @@ -70,6 +70,12 @@ spec: fieldPath: metadata.namespace - name: SEAWEEDFS_FULLNAME value: "{{ template "seaweedfs.name" . }}" + {{- if .Values.master.extraEnvironmentVars }} + {{- range $key, $value := .Values.master.extraEnvironmentVars }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} command: - "/bin/sh" - "-ec" @@ -84,6 +90,11 @@ spec: -port={{ .Values.master.port }} \ -mdir=/data \ -ip.bind={{ .Values.master.ipBind }} \ + {{- if .Values.global.enableReplication }} + -defaultReplication={{ .Values.global.replicationPlacment }} \ + {{- else }} + -defaultReplication={{ .Values.master.defaultReplication }} \ + {{- end }} {{- if .Values.master.volumePreallocate }} -volumePreallocate \ {{- end }} @@ -94,6 +105,15 @@ spec: {{- if .Values.master.disableHttp }} -disableHttp \ {{- end }} + {{- if .Values.master.pulseSeconds }} + -pulseSeconds={{ .Values.master.pulseSeconds }} \ + {{- end }} + {{- if .Values.master.garbageThreshold }} + -garbageThreshold={{ .Values.master.garbageThreshold }} \ + {{- end }} + {{- if .Values.master.metricsIntervalSec }} + -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \ + {{- end }} -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \ -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} volumeMounts: @@ -133,19 +153,21 @@ spec: path: /cluster/status port: {{ .Values.master.port }} scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 15 + initialDelaySeconds: 10 + periodSeconds: 45 successThreshold: 2 failureThreshold: 100 + timeoutSeconds: 5 livenessProbe: httpGet: path: /cluster/status port: {{ .Values.master.port }} scheme: HTTP initialDelaySeconds: 20 - periodSeconds: 10 + periodSeconds: 30 successThreshold: 1 - failureThreshold: 6 + failureThreshold: 4 + timeoutSeconds: 5 {{- if .Values.master.resources }} resources: {{ tpl .Values.master.resources . | nindent 12 | trim }} diff --git a/k8s/seaweedfs/templates/s3-deployment.yaml b/k8s/seaweedfs/templates/s3-deployment.yaml index 1bb3283f1..0e786a6c3 100644 --- a/k8s/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/seaweedfs/templates/s3-deployment.yaml @@ -116,6 +116,7 @@ spec: periodSeconds: 15 successThreshold: 1 failureThreshold: 100 + timeoutSeconds: 3 livenessProbe: httpGet: path: / @@ -125,6 +126,7 @@ spec: periodSeconds: 60 successThreshold: 1 failureThreshold: 20 + timeoutSeconds: 3 {{- if .Values.s3.resources }} resources: {{ tpl .Values.s3.resources . | nindent 12 | trim }} diff --git a/k8s/seaweedfs/templates/seaweefs-grafana-dashboard.yaml b/k8s/seaweedfs/templates/seaweefs-grafana-dashboard.yaml index c943ea50f..d06bafd1c 100644 --- a/k8s/seaweedfs/templates/seaweefs-grafana-dashboard.yaml +++ b/k8s/seaweedfs/templates/seaweefs-grafana-dashboard.yaml @@ -91,7 +91,7 @@ data: "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Filer Request Duration 95th percentile", + "title": "Filer Request Duration 80th percentile", "tooltip": { "msResolution": true, "shared": true, @@ -1349,4 +1349,4 @@ data: "title": "SeaweedFS", "version": 3 } -{{- end }} \ No newline at end of file +{{- end }} diff --git a/k8s/seaweedfs/templates/volume-statefulset.yaml b/k8s/seaweedfs/templates/volume-statefulset.yaml index 9c6ddcd9f..4db8391f9 100644 --- a/k8s/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/seaweedfs/templates/volume-statefulset.yaml @@ -12,6 +12,7 @@ metadata: spec: serviceName: {{ template "seaweedfs.name" . }}-volume replicas: {{ .Values.volume.replicas }} + podManagementPolicy: Parallel selector: matchLabels: app: {{ template "seaweedfs.name" . }} @@ -33,7 +34,7 @@ spec: restartPolicy: {{ default .Values.global.restartPolicy .Values.volume.restartPolicy }} {{- if .Values.volume.tolerations }} tolerations: - {{ tpl .Values.volume.tolerations . | nindent 8 | trim }} + {{ tpl .Values.volume.tolerations . | nindent 8 | trim }} {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: @@ -62,7 +63,7 @@ spec: fieldRef: fieldPath: status.hostIP - name: SEAWEEDFS_FULLNAME - value: "{{ template "seaweedfs.name" . }}" + value: "{{ template "seaweedfs.name" . }}" command: - "/bin/sh" - "-ec" @@ -91,6 +92,16 @@ spec: {{- if .Values.volume.imagesFixOrientation }} -images.fix.orientation \ {{- end }} + {{- if .Values.volume.pulseSeconds }} + -pulseSeconds={{ .Values.volume.pulseSeconds }} \ + {{- end }} + {{- if .Values.volume.index }} + -index={{ .Values.volume.index }} \ + {{- end }} + {{- if .Values.volume.fileSizeLimitMB }} + -fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \ + {{- end }} + -minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \ -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \ -compactionMBps={{ .Values.volume.compactionMBps }} \ -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} @@ -131,19 +142,21 @@ spec: path: /status port: {{ .Values.volume.port }} scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 15 + initialDelaySeconds: 15 + periodSeconds: 90 successThreshold: 1 failureThreshold: 100 + timeoutSeconds: 5 livenessProbe: httpGet: path: /status port: {{ .Values.volume.port }} scheme: HTTP initialDelaySeconds: 20 - periodSeconds: 30 + periodSeconds: 90 successThreshold: 1 - failureThreshold: 10 + failureThreshold: 4 + timeoutSeconds: 5 {{- if .Values.volume.resources }} resources: {{ tpl .Values.volume.resources . | nindent 12 | trim }} diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 4d8d367f3..22163ef60 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -14,6 +14,13 @@ global: enabled: false gatewayHost: null gatewayPort: null + # if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config + enableReplication: false + # replication type is XYZ: + # X number of replica in other data centers + # Y number of replica in other racks in the same data center + # Z number of replica in other servers in the same rack + replicationPlacment: "001" image: registry: "" @@ -31,8 +38,20 @@ master: grpcPort: 19333 ipBind: "0.0.0.0" volumePreallocate: false + #Master stops directing writes to oversized volumes volumeSizeLimitMB: 30000 loggingOverrideLevel: null + #number of seconds between heartbeats, default 5 + pulseSeconds: null + #threshold to vacuum and reclaim spaces, default 0.3 (30%) + garbageThreshold: null + #Prometheus push interval in seconds, default 15 + metricsIntervalSec: 15 + # replication type is XYZ: + # X number of replica in other data centers + # Y number of replica in other racks in the same data center + # Z number of replica in other servers in the same rack + defaultReplication: "000" # Disable http request, only gRpc operations are allowed disableHttp: false @@ -87,6 +106,11 @@ master: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + extraEnvironmentVars: + WEED_MASTER_VOLUME_GROWTH_COPY_1: 7 + WEED_MASTER_VOLUME_GROWTH_COPY_2: 6 + WEED_MASTER_VOLUME_GROWTH_COPY_3: 3 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 volume: enabled: true @@ -100,9 +124,18 @@ volume: ipBind: "0.0.0.0" replicas: 1 loggingOverrideLevel: null + # number of seconds between heartbeats, must be smaller than or equal to the master's setting + pulseSeconds: null + # Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance., default memory + index: null + # limit file size to avoid out of memory, default 256mb + fileSizeLimitMB: null + # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly + minFreeSpacePercent: 1 + # limit background compaction or copying speed in mega bytes per second - compactionMBps: "40" + compactionMBps: "50" # Directories to store data files. dir[,dir]... (default "/tmp") dir: "/data" @@ -177,6 +210,20 @@ filer: port: 8888 grpcPort: 18888 loggingOverrideLevel: null + # replication type is XYZ: + # X number of replica in other data centers + # Y number of replica in other racks in the same data center + # Z number of replica in other servers in the same rack + defaultReplicaPlacement: "000" + # turn off directory listing + disableDirListing: false + # split files larger than the limit, default 32 + maxMB: null + # encrypt data on volume servers + encryptVolumeData: false + + # Whether proxy or redirect to volume server during file GET request + redirectOnRead: false # Limit sub dir listing size (default 100000) dirListLimit: 100000 @@ -237,11 +284,6 @@ filer: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" - dbSchema: - imageName: db-schema - imageTag: "development" - imageOverride: "" - # extraEnvVars is a list of extra enviroment variables to set with the stateful set. extraEnvironmentVars: WEED_MYSQL_ENABLED: "true" @@ -260,6 +302,8 @@ filer: WEED_FILER_BUCKETS_FOLDER: "/buckets" # directories under this folder will be store message queue data WEED_FILER_QUEUES_FOLDER: "/queues" + # WEED_FILER_OPTIONS_BUCKETS_FSYNC a list of buckets names with all write requests fsync=true + WEED_FILER_OPTIONS_BUCKETS_FSYNC: [] s3: enabled: true