From 95787ea89debfc816e7c8f818156f440063d5d8d Mon Sep 17 00:00:00 2001 From: spastorclovr <134076845+spastorclovr@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:25:17 +0100 Subject: [PATCH] Enable multiple disks per volume server. Also streamline the use of /logs & /idx (#5236) * Enable multiple disks per volume server. Also streamline the use of /logs and /idx * missing file * add loop around volume mounts for initContainers * remove uneeded name for idx --- k8s/charts/seaweedfs/templates/_helpers.tpl | 2 +- .../templates/volume-statefulset.yaml | 107 ++++++++++-------- k8s/charts/seaweedfs/values.yaml | 49 ++++---- 3 files changed, 88 insertions(+), 70 deletions(-) diff --git a/k8s/charts/seaweedfs/templates/_helpers.tpl b/k8s/charts/seaweedfs/templates/_helpers.tpl index e9f35447d..f6f7b8991 100644 --- a/k8s/charts/seaweedfs/templates/_helpers.tpl +++ b/k8s/charts/seaweedfs/templates/_helpers.tpl @@ -142,7 +142,7 @@ Inject extra environment vars in the format key:value, if populated {{/* check if any InitContainers exist for Volumes */}} {{- define "volume.initContainers_exists" -}} -{{- if or (not (empty .Values.volume.dir_idx )) (not (empty .Values.volume.initContainers )) -}} +{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}} {{- printf "true" -}} {{- else -}} {{- printf "" -}} diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index e03db4ba7..09c5f7d45 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -54,17 +54,19 @@ spec: {{- $initContainers_exists := include "volume.initContainers_exists" . -}} {{- if $initContainers_exists }} initContainers: - {{- if .Values.volume.dir_idx }} + {{- if .Values.volume.idx }} - name: seaweedfs-vol-move-idx image: {{ template "volume.image" . }} imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} command: [ '/bin/sh', '-c' ] - args: [ 'if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;' ] + args: [ 'ls {{range $dir := .Values.volume.dataDirs }}/{{$dir.name}}/*.idx {{end}} && xargs -I {} mv {} /idx/' ] volumeMounts: - name: idx - mountPath: {{ .Values.volume.dir_idx }} - - name: data - mountPath: {{ .Values.volume.dir }} + mountPath: {{ .Values.volume.idx.name }} + {{- range $dir := .Values.volume.dataDirs }} + - name: {{ $dir.name }} + mountPath: /{{ $dir.name }} + {{- end }} {{- end }} {{- if .Values.volume.initContainers }} {{ tpl .Values.volume.initContainers . | nindent 8 | trim }} @@ -100,7 +102,7 @@ spec: - "-ec" - | exec /usr/bin/weed \ - {{- if eq .Values.volume.logs.type "hostPath" }} + {{- if .Values.volume.logs }} -logdir=/logs \ {{- else }} -logtostderr=true \ @@ -115,11 +117,11 @@ spec: {{- if .Values.volume.metricsPort }} -metricsPort={{ .Values.volume.metricsPort }} \ {{- end }} - -dir={{ .Values.volume.dir }} \ - {{- if .Values.volume.dir_idx }} - -dir.idx={{ .Values.volume.dir_idx }} \ + -dir {{range $index, $dir := .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}/{{$dir.name}}{{end}} \ + {{- if .Values.volume.idx }} + -dir.idx= /idx \ {{- end }} - -max={{ .Values.volume.maxVolumes }} \ + -max {{range $index, $dir := .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}{{$dir.maxVolumes}}{{end}} \ {{- if .Values.volume.rack }} -rack={{ .Values.volume.rack }} \ {{- end }} @@ -148,16 +150,18 @@ spec: -compactionMBps={{ .Values.volume.compactionMBps }} \ -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} volumeMounts: - - name: data - mountPath: "{{ .Values.volume.dir }}/" - {{- if .Values.volume.dir_idx }} - - name: idx - mountPath: "{{ .Values.volume.dir_idx }}/" + {{- range $dir := .Values.volume.dataDirs }} + - name: {{ $dir.name }} + mountPath: "/{{ $dir.name }}/" {{- end }} - {{- if eq .Values.volume.logs.type "hostPath" }} + {{- if .Values.volume.logs }} - name: logs mountPath: "/logs/" {{- end }} + {{- if .Values.volume.idx }} + - name: idx + mountPath: "/idx/" + {{- end }} {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -221,40 +225,51 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }} {{- end }} volumes: - {{- if eq .Values.volume.data.type "hostPath" }} - - name: data + + {{- range $dir := .Values.volume.dataDirs }} + + {{- if eq $dir.type "hostPath" }} + - name: {{ $dir.name }} hostPath: - path: {{ .Values.volume.data.hostPathPrefix }}/object_store/ + path: {{ $dir.hostPathPrefix }}/object_store/ type: DirectoryOrCreate - {{- end }} - {{- if eq .Values.volume.data.type "existingClaim" }} - - name: data + {{- end }} + {{- if eq $dir.type "existingClaim" }} + - name: {{ $dir.name }} persistentVolumeClaim: - claimName: {{ .Values.volume.data.claimName }} - {{- end }} - {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }} + claimName: {{ $dir.claimName }} + {{- end }} + + {{- end }} + + {{- if .Values.volume.idx }} + {{- if eq .Values.volume.idx.type "hostPath" }} - name: idx hostPath: path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/ type: DirectoryOrCreate - {{- end }} - {{- if eq .Values.volume.idx.type "existingClaim" }} + {{- end }} + {{- if eq .Values.volume.idx.type "existingClaim" }} - name: idx persistentVolumeClaim: claimName: {{ .Values.volume.idx.claimName }} - {{- end }} - {{- if eq .Values.volume.logs.type "hostPath" }} + {{- end }} + {{- end }} + + {{- if .Values.volume.logs }} + {{- if eq .Values.volume.logs.type "hostPath" }} - name: logs hostPath: path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume type: DirectoryOrCreate - {{- end }} - {{- if eq .Values.volume.logs.type "existingClaim" }} + {{- end }} + {{- if eq .Values.volume.logs.type "existingClaim" }} - name: logs persistentVolumeClaim: - claimName: {{ .Values.volume.data.claimName }} - {{- end }} - {{- if .Values.global.enableSecurity }} + claimName: {{ .Values.volume.logs.claimName }} + {{- end }} + {{- end }} + {{- if .Values.global.enableSecurity }} - name: security-config configMap: name: {{ template "seaweedfs.name" . }}-security-config @@ -273,7 +288,7 @@ spec: - name: client-cert secret: secretName: {{ template "seaweedfs.name" . }}-client-cert - {{- end }} + {{- end }} {{- if .Values.volume.extraVolumes }} {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }} {{- end }} @@ -281,24 +296,25 @@ spec: nodeSelector: {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }} {{- end }} - {{- $pvc_exists := include "volume.pvc_exists" . -}} - {{- if $pvc_exists }} volumeClaimTemplates: - {{- if eq .Values.volume.data.type "persistentVolumeClaim"}} + {{- range $dir := .Values.volume.dataDirs }} + {{- if eq $dir.type "persistentVolumeClaim" }} - metadata: - name: data - {{- with .Values.volume.data.annotations }} + name: {{ $dir.name }} + {{- with $dir.annotations }} annotations: {{- toYaml . | nindent 10 }} {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.volume.data.storageClass }} + storageClassName: {{ $dir.storageClass }} resources: requests: - storage: {{ .Values.volume.data.size }} + storage: {{ $dir.size }} + {{- end }} {{- end }} - {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }} + + {{- if and .Values.volume.idx (eq .Values.volume.idx.type "persistentVolumeClaim") }} - metadata: name: idx {{- with .Values.volume.idx.annotations }} @@ -312,7 +328,7 @@ spec: requests: storage: {{ .Values.volume.idx.size }} {{- end }} - {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }} + {{- if and .Values.volume.logs (eq .Values.volume.logs.type "persistentVolumeClaim") }} - metadata: name: logs {{- with .Values.volume.logs.annotations }} @@ -326,5 +342,4 @@ spec: requests: storage: {{ .Values.volume.logs.size }} {{- end }} - {{- end }} -{{- end }} + {{- end }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index a0138d15c..b0bcaf837 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -89,7 +89,6 @@ master: # claimName: "my-pvc" data: type: "hostPath" - size: "" storageClass: "" hostPathPrefix: /ssd @@ -239,25 +238,36 @@ volume: # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly minFreeSpacePercent: 7 - # You may use ANY storage-class, example with local-path-provisioner + # For each data disk you may use ANY storage-class, example with local-path-provisioner # Annotations are optional. - # data: - # type: "persistentVolumeClaim" - # size: "24Ti" - # storageClass: "local-path-provisioner" - # annotations: + # dataDirs: + # - name: data: + # type: "persistentVolumeClaim" + # size: "24Ti" + # storageClass: "local-path-provisioner" + # annotations: # "key": "value" + # maxVolumes: 0 # # You may also spacify an existing claim: - # data: - # type: "existingClaim" - # claimName: "my-pvc" + # - name: data + # type: "existingClaim" + # claimName: "my-pvc" + # maxVolumes: 0 + + + dataDirs: + - name: data1 + type: "hostPath" + hostPathPrefix: /ssd + maxVolumes: 0 # If set to zero on non-windows OS, the limit will be auto configured. (default "7") + + #- name: data2 + # type: "persistentVolumeClaim" + # storageClass: "yourClassNameOfChoice" + # size: "800Gi" + # maxVolumes: 0 - data: - type: "hostPath" - size: "" - storageClass: "" - hostPathPrefix: /storage idx: type: "hostPath" @@ -265,6 +275,7 @@ volume: storageClass: "" hostPathPrefix: /ssd + logs: type: "hostPath" size: "" @@ -274,14 +285,6 @@ volume: # limit background compaction or copying speed in mega bytes per second compactionMBps: "50" - # Directories to store data files. dir[,dir]... (default "/tmp") - dir: "/data" - # Directories to store index files. dir[,dir]... (default is the same as "dir") - dir_idx: null - - # Maximum numbers of volumes, count[,count]... - # If set to zero on non-windows OS, the limit will be auto configured. (default "7") - maxVolumes: "0" # Volume server's rack name rack: null