Enable multiple disks per volume server. Also streamline the use of /logs & /idx (#5236)

* Enable multiple disks per volume server. Also streamline the use of /logs and /idx

* missing file

* add loop around volume mounts for initContainers

* remove uneeded name for idx

k8s/charts/seaweedfs/templates/_helpers.tpl

@@ -142,7 +142,7 @@ Inject extra environment vars in the format key:value, if populated
 
 {{/* check if any InitContainers exist for Volumes */}}
 {{- define "volume.initContainers_exists" -}}
-{{- if or (not (empty .Values.volume.dir_idx )) (not (empty .Values.volume.initContainers )) -}}
+{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
 {{- printf "true" -}}
 {{- else -}}
 {{- printf "" -}}

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -54,17 +54,19 @@ spec:
       {{- $initContainers_exists := include "volume.initContainers_exists" . -}}
       {{- if $initContainers_exists }}
       initContainers:
-        {{- if .Values.volume.dir_idx }}
+        {{- if .Values.volume.idx }}
         - name: seaweedfs-vol-move-idx
           image: {{ template "volume.image" . }}
           imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }}
           command: [ '/bin/sh', '-c' ]
-          args: [ 'if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;' ]
+          args: [ 'ls {{range $dir :=  .Values.volume.dataDirs }}/{{$dir.name}}/*.idx {{end}} && xargs -I {} mv {} /idx/' ]
           volumeMounts:
             - name: idx
-              mountPath: {{ .Values.volume.dir_idx }}
-            - name: data
-              mountPath: {{ .Values.volume.dir }}
+              mountPath: {{ .Values.volume.idx.name }}
+          {{- range $dir :=  .Values.volume.dataDirs }} 
+            - name: {{ $dir.name }}
+              mountPath: /{{ $dir.name }}
+          {{- end }}
         {{- end }}
         {{- if .Values.volume.initContainers }}
         {{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
@@ -100,7 +102,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-                {{- if eq .Values.volume.logs.type "hostPath" }}
+                {{- if .Values.volume.logs }}
                 -logdir=/logs \
                 {{- else }}
                 -logtostderr=true \
@@ -115,11 +117,11 @@ spec:
                 {{- if .Values.volume.metricsPort }}
                 -metricsPort={{ .Values.volume.metricsPort }} \
                 {{- end }}
-                -dir={{ .Values.volume.dir }} \
-                {{- if .Values.volume.dir_idx }}
-                -dir.idx={{ .Values.volume.dir_idx }} \
+                -dir {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}/{{$dir.name}}{{end}} \
+                {{- if .Values.volume.idx }}
+                -dir.idx= /idx \
                 {{- end }}
-                -max={{ .Values.volume.maxVolumes }} \
+                -max {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}{{$dir.maxVolumes}}{{end}} \
                 {{- if .Values.volume.rack }}
                 -rack={{ .Values.volume.rack }} \
                 {{- end }}
@@ -148,16 +150,18 @@ spec:
                 -compactionMBps={{ .Values.volume.compactionMBps }} \
                 -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           volumeMounts:
-            - name: data
-              mountPath: "{{ .Values.volume.dir }}/"
-            {{- if .Values.volume.dir_idx }}
-            - name: idx
-              mountPath: "{{ .Values.volume.dir_idx }}/"
+            {{- range $dir := .Values.volume.dataDirs }}
+            - name: {{ $dir.name }}
+              mountPath: "/{{ $dir.name }}/"
             {{- end }}
-            {{- if eq .Values.volume.logs.type "hostPath" }}
+            {{- if .Values.volume.logs }}
             - name: logs
               mountPath: "/logs/"
             {{- end }}
+            {{- if .Values.volume.idx }}
+            - name: idx
+              mountPath: "/idx/"
+            {{- end }}
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -221,40 +225,51 @@ spec:
       {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
       {{- end }}
       volumes:
-        {{- if eq .Values.volume.data.type "hostPath" }}
-        - name: data
+
+     {{- range $dir := .Values.volume.dataDirs }}
+
+      {{- if eq $dir.type "hostPath" }}
+        - name: {{ $dir.name }}
           hostPath:
-            path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
+            path: {{ $dir.hostPathPrefix }}/object_store/
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.data.type "existingClaim" }}
-        - name: data
+      {{- end }}
+      {{- if eq $dir.type "existingClaim" }}
+        - name: {{ $dir.name }}
           persistentVolumeClaim:
-            claimName: {{ .Values.volume.data.claimName }} 
-        {{- end }}
-        {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
+            claimName: {{ $dir.claimName }} 
+      {{- end }}
+
+     {{- end }}
+
+     {{- if .Values.volume.idx }}
+       {{- if eq .Values.volume.idx.type "hostPath" }}
         - name: idx
           hostPath:
             path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.idx.type "existingClaim" }}
+       {{- end }}
+       {{- if eq .Values.volume.idx.type "existingClaim" }}
         - name: idx
           persistentVolumeClaim:
             claimName: {{ .Values.volume.idx.claimName }} 
-        {{- end }}
-        {{- if eq .Values.volume.logs.type "hostPath" }}
+       {{- end }}
+     {{- end }}
+
+     {{- if .Values.volume.logs }}
+       {{- if eq .Values.volume.logs.type "hostPath" }}
         - name: logs
           hostPath:
             path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.logs.type "existingClaim" }}
+       {{- end }}
+       {{- if eq .Values.volume.logs.type "existingClaim" }}
         - name: logs
           persistentVolumeClaim:
-            claimName: {{ .Values.volume.data.claimName }} 
-        {{- end }}
-        {{- if .Values.global.enableSecurity }}
+            claimName: {{ .Values.volume.logs.claimName }} 
+       {{- end }}
+     {{- end }}
+     {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
             name: {{ template "seaweedfs.name" . }}-security-config
@@ -273,7 +288,7 @@ spec:
         - name: client-cert
           secret:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
-        {{- end }}
+      {{- end }}
       {{- if .Values.volume.extraVolumes }}
         {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
       {{- end }}
@@ -281,24 +296,25 @@ spec:
       nodeSelector:
         {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
       {{- end }}
-  {{- $pvc_exists := include "volume.pvc_exists" . -}}
-  {{- if $pvc_exists }}
   volumeClaimTemplates:
-    {{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
+    {{- range $dir := .Values.volume.dataDirs }}
+    {{- if eq $dir.type "persistentVolumeClaim" }}
     - metadata:
-        name: data
-        {{- with .Values.volume.data.annotations }}
+        name: {{ $dir.name }}
+        {{- with $dir.annotations }}
         annotations:
           {{- toYaml . | nindent 10 }}
         {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
-        storageClassName: {{ .Values.volume.data.storageClass }}
+        storageClassName: {{ $dir.storageClass }}
         resources:
           requests:
-            storage: {{ .Values.volume.data.size }}
+            storage: {{ $dir.size }}
+    {{- end }}
     {{- end }}
-    {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
+
+    {{- if and .Values.volume.idx (eq .Values.volume.idx.type "persistentVolumeClaim") }}
     - metadata:
         name: idx
         {{- with .Values.volume.idx.annotations }}
@@ -312,7 +328,7 @@ spec:
           requests:
             storage: {{ .Values.volume.idx.size }}
     {{- end }}
-    {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
+    {{- if and .Values.volume.logs (eq .Values.volume.logs.type "persistentVolumeClaim") }}
     - metadata:
         name: logs
         {{- with .Values.volume.logs.annotations }}
@@ -326,5 +342,4 @@ spec:
           requests:
             storage: {{ .Values.volume.logs.size }}
     {{- end }}
-      {{- end }}
-{{- end }}
+    {{- end }}

k8s/charts/seaweedfs/values.yaml

@@ -89,7 +89,6 @@ master:
   #    claimName: "my-pvc"
   data:
     type: "hostPath"
-    size: ""
     storageClass: ""
     hostPathPrefix: /ssd
 
@@ -239,25 +238,36 @@ volume:
   # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
   minFreeSpacePercent: 7
 
-  # You may use ANY storage-class, example with local-path-provisioner
+  # For each data disk you may use ANY storage-class, example with local-path-provisioner
   # Annotations are optional.
-  #  data:
-  #    type: "persistentVolumeClaim"
-  #    size: "24Ti"
-  #    storageClass: "local-path-provisioner"
-  #    annotations:
+  #  dataDirs:
+  #   - name: data:
+  #     type: "persistentVolumeClaim"
+  #     size: "24Ti"
+  #     storageClass: "local-path-provisioner"
+  #     annotations:
   #      "key": "value"
+  #     maxVolumes: 0
   #
   # You may also spacify an existing claim:
-  #  data:
-  #    type: "existingClaim"
-  #    claimName: "my-pvc"
+  #    - name: data
+  #      type: "existingClaim"
+  #      claimName: "my-pvc"
+  #     maxVolumes: 0
+
+
+  dataDirs:
+   - name: data1
+     type: "hostPath"
+     hostPathPrefix: /ssd
+     maxVolumes: 0   # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
+
+   #- name: data2
+   #  type: "persistentVolumeClaim"
+   #  storageClass: "yourClassNameOfChoice"
+   #  size: "800Gi"
+   #  maxVolumes: 0
 
-  data:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-    hostPathPrefix: /storage
 
   idx:
     type: "hostPath"
@@ -265,6 +275,7 @@ volume:
     storageClass: ""
     hostPathPrefix: /ssd
 
+
   logs:
     type: "hostPath"
     size: ""
@@ -274,14 +285,6 @@ volume:
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
 
-  # Directories to store data files. dir[,dir]... (default "/tmp")
-  dir: "/data"
-  # Directories to store index files. dir[,dir]... (default is the same as "dir")
-  dir_idx: null
-
-  # Maximum numbers of volumes, count[,count]...
-  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
-  maxVolumes: "0"
 
   # Volume server's rack name
   rack: null