From 8400e102427f8347933b02e41e3b5fd76b3f5cad Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 25 Oct 2025 18:38:56 -0700 Subject: [PATCH] workers are stateless, admin is stateful --- k8s/charts/seaweedfs/README.md | 46 ++- .../templates/admin/admin-deployment.yaml | 4 +- .../templates/admin/admin-statefulset.yaml | 317 ++++++++++++++++++ .../templates/worker/worker-deployment.yaml | 43 +-- .../templates/worker/worker-service.yaml | 27 ++ k8s/charts/seaweedfs/values.yaml | 5 + 6 files changed, 400 insertions(+), 42 deletions(-) create mode 100644 k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml create mode 100644 k8s/charts/seaweedfs/templates/worker/worker-service.yaml diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index ccc0e87c5..f75c4feae 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -255,18 +255,43 @@ For production deployments, consider: Example specialized worker configuration: +For specialized worker pools, deploy separate Helm releases with different capabilities: + +**values-worker-vacuum.yaml** (for vacuum operations): ```yaml -# In your values.yaml - deploy multiple worker deployments manually -# One for vacuum operations +# Disable all other components, enable only workers +master: + enabled: false +volume: + enabled: false +filer: + enabled: false +s3: + enabled: false +admin: + enabled: false + worker: enabled: true replicas: 2 capabilities: "vacuum" maxConcurrent: 2 - -# Deploy another release for balance operations -# helm install seaweedfs-worker-balance seaweedfs/seaweedfs --values worker-balance-values.yaml -# worker-balance-values.yaml: +``` + +**values-worker-balance.yaml** (for balance operations): +```yaml +# Disable all other components, enable only workers +master: + enabled: false +volume: + enabled: false +filer: + enabled: false +s3: + enabled: false +admin: + enabled: false + worker: enabled: true replicas: 1 @@ -274,6 +299,15 @@ worker: maxConcurrent: 1 ``` +Deploy the specialized workers as separate releases: +```bash +# Deploy vacuum workers +helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum.yaml + +# Deploy balance workers +helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml +``` + ## Enterprise For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition, diff --git a/k8s/charts/seaweedfs/templates/admin/admin-deployment.yaml b/k8s/charts/seaweedfs/templates/admin/admin-deployment.yaml index 826eefb17..4e922ced3 100644 --- a/k8s/charts/seaweedfs/templates/admin/admin-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/admin/admin-deployment.yaml @@ -1,6 +1,6 @@ {{- if .Values.admin.enabled }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ template "seaweedfs.name" . }}-admin namespace: {{ .Release.Namespace }} @@ -15,6 +15,8 @@ metadata: {{- toYaml .Values.admin.annotations | nindent 4 }} {{- end }} spec: + serviceName: {{ template "seaweedfs.name" . }}-admin + podManagementPolicy: {{ .Values.admin.podManagementPolicy }} replicas: {{ .Values.admin.replicas }} selector: matchLabels: diff --git a/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml b/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml new file mode 100644 index 000000000..4e922ced3 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml @@ -0,0 +1,317 @@ +{{- if .Values.admin.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "seaweedfs.name" . }}-admin + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admin +{{- if .Values.admin.annotations }} + annotations: + {{- toYaml .Values.admin.annotations | nindent 4 }} +{{- end }} +spec: + serviceName: {{ template "seaweedfs.name" . }}-admin + podManagementPolicy: {{ .Values.admin.podManagementPolicy }} + replicas: {{ .Values.admin.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admin + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admin + {{ with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.admin.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{ with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.admin.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + restartPolicy: {{ default .Values.global.restartPolicy .Values.admin.restartPolicy }} + {{- if .Values.admin.affinity }} + affinity: + {{ tpl .Values.admin.affinity . | nindent 8 | trim }} + {{- end }} + {{- if .Values.admin.topologySpreadConstraints }} + topologySpreadConstraints: + {{ tpl .Values.admin.topologySpreadConstraints . | nindent 8 | trim }} + {{- end }} + {{- if .Values.admin.tolerations }} + tolerations: + {{ tpl .Values.admin.tolerations . | nindent 8 | trim }} + {{- end }} + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} + terminationGracePeriodSeconds: 30 + {{- if .Values.admin.priorityClassName }} + priorityClassName: {{ .Values.admin.priorityClassName | quote }} + {{- end }} + enableServiceLinks: false + {{- if .Values.admin.serviceAccountName }} + serviceAccountName: {{ .Values.admin.serviceAccountName | quote }} + {{- end }} + {{- if .Values.admin.initContainers }} + initContainers: + {{ tpl .Values.admin.initContainers . | nindent 8 | trim }} + {{- end }} + {{- if .Values.admin.podSecurityContext.enabled }} + securityContext: {{- omit .Values.admin.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + containers: + - name: seaweedfs + image: {{ template "admin.image" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SEAWEEDFS_FULLNAME + value: "{{ template "seaweedfs.name" . }}" + {{- if .Values.admin.extraEnvironmentVars }} + {{- range $key, $value := .Values.admin.extraEnvironmentVars }} + - name: {{ $key }} + {{- if kindIs "string" $value }} + value: {{ $value | quote }} + {{- else }} + valueFrom: + {{ toYaml $value | nindent 16 | trim }} + {{- end -}} + {{- end }} + {{- end }} + {{- if .Values.global.extraEnvironmentVars }} + {{- range $key, $value := .Values.global.extraEnvironmentVars }} + - name: {{ $key }} + {{- if kindIs "string" $value }} + value: {{ $value | quote }} + {{- else }} + valueFrom: + {{ toYaml $value | nindent 16 | trim }} + {{- end -}} + {{- end }} + {{- end }} + command: + - "/bin/sh" + - "-ec" + - | + exec /usr/bin/weed \ + {{- if or (eq .Values.admin.logs.type "hostPath") (eq .Values.admin.logs.type "persistentVolumeClaim") (eq .Values.admin.logs.type "emptyDir") }} + -logdir=/logs \ + {{- else }} + -logtostderr=true \ + {{- end }} + {{- if .Values.admin.loggingOverrideLevel }} + -v={{ .Values.admin.loggingOverrideLevel }} \ + {{- else }} + -v={{ .Values.global.loggingLevel }} \ + {{- end }} + admin \ + -port={{ .Values.admin.port }} \ + -port.grpc={{ .Values.admin.grpcPort }} \ + {{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }} + -dataDir=/data \ + {{- else if .Values.admin.dataDir }} + -dataDir={{ .Values.admin.dataDir }} \ + {{- end }} + {{- if .Values.admin.adminPassword }} + -adminUser={{ .Values.admin.adminUser }} \ + -adminPassword={{ .Values.admin.adminPassword }} \ + {{- end }} + {{- if .Values.admin.masters }} + -masters={{ .Values.admin.masters }} \ + {{- else if .Values.global.masterServer }} + -masters={{ .Values.global.masterServer }} \ + {{- else }} + -masters={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} \ + {{- end }} + {{- range .Values.admin.extraArgs }} + {{ . }} \ + {{- end }} + volumeMounts: + {{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }} + - name: admin-data + mountPath: /data + {{- end }} + {{- if or (eq .Values.admin.logs.type "hostPath") (eq .Values.admin.logs.type "persistentVolumeClaim") (eq .Values.admin.logs.type "emptyDir") (eq .Values.admin.logs.type "existingClaim") }} + - name: admin-logs + mountPath: /logs + {{- end }} + {{- if .Values.global.enableSecurity }} + - name: security-config + readOnly: true + mountPath: /etc/seaweedfs/security.toml + subPath: security.toml + - name: ca-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/ca/ + - name: master-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/master/ + - name: volume-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/volume/ + - name: filer-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/filer/ + - name: client-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/client/ + {{- end }} + {{ tpl .Values.admin.extraVolumeMounts . | nindent 12 | trim }} + ports: + - containerPort: {{ .Values.admin.port }} + name: http + - containerPort: {{ .Values.admin.grpcPort }} + name: grpc + {{- if .Values.admin.metricsPort }} + - containerPort: {{ .Values.admin.metricsPort }} + name: metrics + {{- end }} + {{- if .Values.admin.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: {{ .Values.admin.readinessProbe.httpGet.path }} + port: {{ .Values.admin.port }} + scheme: {{ .Values.admin.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.admin.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.admin.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.admin.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.admin.readinessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.admin.readinessProbe.timeoutSeconds }} + {{- end }} + {{- if .Values.admin.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: {{ .Values.admin.livenessProbe.httpGet.path }} + port: {{ .Values.admin.port }} + scheme: {{ .Values.admin.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.admin.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.admin.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.admin.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.admin.livenessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.admin.livenessProbe.timeoutSeconds }} + {{- end }} + {{- with .Values.admin.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.admin.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.admin.sidecars }} + {{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + {{- if eq .Values.admin.data.type "hostPath" }} + - name: admin-data + hostPath: + path: {{ .Values.admin.data.hostPathPrefix }}/seaweedfs-admin-data + type: DirectoryOrCreate + {{- end }} + {{- if eq .Values.admin.data.type "emptyDir" }} + - name: admin-data + emptyDir: {} + {{- end }} + {{- if eq .Values.admin.data.type "existingClaim" }} + - name: admin-data + persistentVolumeClaim: + claimName: {{ .Values.admin.data.claimName }} + {{- end }} + {{- if eq .Values.admin.logs.type "hostPath" }} + - name: admin-logs + hostPath: + path: {{ .Values.admin.logs.hostPathPrefix }}/logs/seaweedfs/admin + type: DirectoryOrCreate + {{- end }} + {{- if eq .Values.admin.logs.type "emptyDir" }} + - name: admin-logs + emptyDir: {} + {{- end }} + {{- if eq .Values.admin.logs.type "existingClaim" }} + - name: admin-logs + persistentVolumeClaim: + claimName: {{ .Values.admin.logs.claimName }} + {{- end }} + {{- if .Values.global.enableSecurity }} + - name: security-config + configMap: + name: {{ template "seaweedfs.name" . }}-security-config + - name: ca-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-ca-cert + - name: master-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-master-cert + - name: volume-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-volume-cert + - name: filer-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-filer-cert + - name: client-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-client-cert + {{- end }} + {{ tpl .Values.admin.extraVolumes . | indent 8 | trim }} + {{- if .Values.admin.nodeSelector }} + nodeSelector: + {{ tpl .Values.admin.nodeSelector . | indent 8 | trim }} + {{- end }} + {{- $pvc_exists := include "admin.pvc_exists" . -}} + {{- if $pvc_exists }} + volumeClaimTemplates: + {{- if eq .Values.admin.data.type "persistentVolumeClaim" }} + - metadata: + name: admin-data + {{- with .Values.admin.data.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.admin.data.storageClass }} + resources: + requests: + storage: {{ .Values.admin.data.size }} + {{- end }} + {{- if eq .Values.admin.logs.type "persistentVolumeClaim" }} + - metadata: + name: admin-logs + {{- with .Values.admin.logs.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.admin.logs.storageClass }} + resources: + requests: + storage: {{ .Values.admin.logs.size }} + {{- end }} + {{- end }} +{{- end }} + diff --git a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml index 1c1dd5f29..00994d0ec 100644 --- a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml @@ -117,7 +117,7 @@ spec: - "-ec" - | exec /usr/bin/weed \ - {{- if or (eq .Values.worker.logs.type "hostPath") (eq .Values.worker.logs.type "persistentVolumeClaim") (eq .Values.worker.logs.type "emptyDir") }} + {{- if or (eq .Values.worker.logs.type "hostPath") (eq .Values.worker.logs.type "emptyDir") }} -logdir=/logs \ {{- else }} -logtostderr=true \ @@ -140,11 +140,11 @@ spec: {{ . }} \ {{- end }} volumeMounts: - {{- if or (eq .Values.worker.data.type "hostPath") (eq .Values.worker.data.type "persistentVolumeClaim") (eq .Values.worker.data.type "emptyDir") (eq .Values.worker.data.type "existingClaim") }} + {{- if or (eq .Values.worker.data.type "hostPath") (eq .Values.worker.data.type "emptyDir") (eq .Values.worker.data.type "existingClaim") }} - name: worker-data mountPath: {{ .Values.worker.workingDir }} {{- end }} - {{- if or (eq .Values.worker.logs.type "hostPath") (eq .Values.worker.logs.type "persistentVolumeClaim") (eq .Values.worker.logs.type "emptyDir") (eq .Values.worker.logs.type "existingClaim") }} + {{- if or (eq .Values.worker.logs.type "hostPath") (eq .Values.worker.logs.type "emptyDir") (eq .Values.worker.logs.type "existingClaim") }} - name: worker-logs mountPath: /logs {{- end }} @@ -170,6 +170,11 @@ spec: mountPath: /usr/local/share/ca-certificates/client/ {{- end }} {{ tpl .Values.worker.extraVolumeMounts . | nindent 12 | trim }} + ports: + {{- if .Values.worker.metricsPort }} + - containerPort: {{ .Values.worker.metricsPort }} + name: metrics + {{- end }} {{- with .Values.worker.resources }} resources: {{- toYaml . | nindent 12 }} @@ -236,37 +241,5 @@ spec: nodeSelector: {{ tpl .Values.worker.nodeSelector . | indent 8 | trim }} {{- end }} - {{- $pvc_exists := include "worker.pvc_exists" . -}} - {{- if $pvc_exists }} - volumeClaimTemplates: - {{- if eq .Values.worker.data.type "persistentVolumeClaim" }} - - metadata: - name: worker-data - {{- with .Values.worker.data.annotations }} - annotations: - {{- toYaml . | nindent 10 }} - {{- end }} - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.worker.data.storageClass }} - resources: - requests: - storage: {{ .Values.worker.data.size }} - {{- end }} - {{- if eq .Values.worker.logs.type "persistentVolumeClaim" }} - - metadata: - name: worker-logs - {{- with .Values.worker.logs.annotations }} - annotations: - {{- toYaml . | nindent 10 }} - {{- end }} - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.worker.logs.storageClass }} - resources: - requests: - storage: {{ .Values.worker.logs.size }} - {{- end }} - {{- end }} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/worker/worker-service.yaml b/k8s/charts/seaweedfs/templates/worker/worker-service.yaml new file mode 100644 index 000000000..8faf5e1ab --- /dev/null +++ b/k8s/charts/seaweedfs/templates/worker/worker-service.yaml @@ -0,0 +1,27 @@ +{{- if .Values.worker.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "seaweedfs.name" . }}-worker + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: worker +spec: + clusterIP: None # Headless service + ports: + {{- if .Values.worker.metricsPort }} + - name: "metrics" + port: {{ .Values.worker.metricsPort }} + targetPort: {{ .Values.worker.metricsPort }} + protocol: TCP + {{- end }} + selector: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: worker +{{- end }} + diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 873968c77..7ebe3617c 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -1141,6 +1141,10 @@ admin: podLabels: {} podAnnotations: {} annotations: {} + + ## Set podManagementPolicy + podManagementPolicy: Parallel + resources: {} tolerations: "" nodeSelector: "" @@ -1194,6 +1198,7 @@ worker: restartPolicy: null replicas: 1 loggingOverrideLevel: null + metricsPort: 9327 # Admin server to connect to # Format: "host:port" or auto-discover from admin service