From f45795694874c702695d7062691c09b62f07f47c Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 22 Feb 2023 11:31:13 -0600 Subject: [PATCH] Helm Chart Documentation and CI Updates (#4243) --- .github/workflows/helm_ci.yml | 51 +++++++++++++++++ k8s/charts/seaweedfs/README.md | 55 +++++++++++++------ .../templates/volume-statefulset.yaml | 2 +- k8s/charts/seaweedfs/values.yaml | 46 ++++++++-------- 4 files changed, 112 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/helm_ci.yml diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml new file mode 100644 index 000000000..6c8843dd7 --- /dev/null +++ b/.github/workflows/helm_ci.yml @@ -0,0 +1,51 @@ +name: "helm: lint and test charts" + +on: + push: + branches: [ master ] + paths: ['k8s/**'] + pull_request: + branches: [ master ] + paths: ['k8s/**'] + +permissions: + contents: read + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.0 + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.3.1 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --chart-dirs k8s/charts) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Run chart-testing (lint) + run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts + + - name: Create kind cluster + uses: helm/kind-action@v1.4.0 + + - name: Run chart-testing (install) + run: ct install --all --chart-dirs k8s/charts \ No newline at end of file diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index a510b02ca..ead85121e 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -1,17 +1,34 @@ -# SEAWEEDFS - helm chart (2.x) +# SEAWEEDFS - helm chart (2.x+) + +## Getting Started + +### Add the helm repo + +`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm` + +### Install the helm chart + +`helm install seaweedfs seaweedfs/seaweedfs` + +### (Recommended) Provide `values.yaml` + +`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs` ## Info: * master/filer/volume are stateful sets with anti-affinity on the hostname, so your deployment will be spread/HA. -* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) -and backup/HA memsql can provide. -* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer -with ENV. -* cert config exists and can be enabled, but not been tested. +* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide. +* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer with ENV. +* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed. ## Prerequisites ### Database -A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. + +leveldb is the default database this only supports one filer replica. + +To have multiple filers a external datastore is recommened. + +Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. This database should be pre-configured and initialized by running: ```sql CREATE TABLE IF NOT EXISTS `filemeta` ( @@ -23,10 +40,12 @@ CREATE TABLE IF NOT EXISTS `filemeta` ( ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; ``` -Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`. +Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`. ### Node Labels -Kubernetes node have labels which help to define which node(Host) will run which pod: +Kubernetes nodes can have labels which help to define which node(Host) will run which pod: + +Here is an example: * s3/filer/master needs the label **sw-backend=true** * volume need the label **sw-volume=true** @@ -36,16 +55,16 @@ kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true ``` on production k8s deployment you will want each pod to have a different host, -especially the volume server & the masters, currently all pods (master/volume/filer) -have anti-affinity rule to disallow running multiple pod type on the same host. -if you still want to run multiple pods of the same type (master/volume/filer) on the same host -please set/update the corresponding affinity rule in values.yaml to an empty one: +especially the volume server and the masters, all pods (master/volume/filer) +should have anti-affinity rules to disallow running multiple component pods on the same host. + +If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one: ```affinity: ""``` ## PVC - storage class ### -on the volume stateful set added support for K8S PVC, currently example +On the volume stateful set added support for k8s PVC, currently example with the simple local-path-provisioner from Rancher (comes included with k3d / k3s) https://github.com/rancher/local-path-provisioner @@ -53,10 +72,10 @@ you can use ANY storage class you like, just update the correct storage-class for your deployment. ## current instances config (AIO): -1 instance for each type (master/filer+s3/volume) -you can update the replicas count for each node type in values.yaml, -need to add more nodes with the corresponding labels. +1 instance for each type (master/filer+s3/volume) -most of the configuration are available through values.yaml +You can update the replicas count for each node type in values.yaml, +need to add more nodes with the corresponding labels if applicable. +Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing). \ No newline at end of file diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 41946e2b3..4c4681618 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -187,7 +187,7 @@ spec: port: {{ .Values.volume.port }} scheme: HTTP initialDelaySeconds: 15 - periodSeconds: 90 + periodSeconds: 15 successThreshold: 1 failureThreshold: 100 timeoutSeconds: 30 diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index d14058a81..ed0f582cf 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -5,7 +5,7 @@ global: repository: "" imageName: chrislusf/seaweedfs imagePullPolicy: IfNotPresent - imagePullSecrets: imagepullsecret + imagePullSecrets: "" restartPolicy: Always loggingLevel: 1 enableSecurity: false @@ -44,11 +44,11 @@ master: volumePreallocate: false volumeSizeLimitMB: 1000 loggingOverrideLevel: null - #number of seconds between heartbeats, default 5 + # number of seconds between heartbeats, default 5 pulseSeconds: null - #threshold to vacuum and reclaim spaces, default 0.3 (30%) + # threshold to vacuum and reclaim spaces, default 0.3 (30%) garbageThreshold: null - #Prometheus push interval in seconds, default 15 + # Prometheus push interval in seconds, default 15 metricsIntervalSec: 15 # replication type is XYZ: # X number of replica in other data centers @@ -112,10 +112,10 @@ master: # nodeSelector labels for master pod assignment, formatted as a muli-line string. # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector # Example: - # nodeSelector: | - # beta.kubernetes.io/arch: amd64 nodeSelector: | - sw-backend: "true" + beta.kubernetes.io/arch: amd64 + # nodeSelector: | + # sw-backend: "true" # used to assign priority to master pods # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ @@ -251,10 +251,10 @@ volume: # nodeSelector labels for server pod assignment, formatted as a muli-line string. # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector # Example: - # nodeSelector: | - # beta.kubernetes.io/arch: amd64 nodeSelector: | - sw-volume: "true" + beta.kubernetes.io/arch: amd64 + # nodeSelector: | + # sw-volume: "true" # used to assign priority to server pods # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ @@ -358,10 +358,10 @@ filer: # nodeSelector labels for server pod assignment, formatted as a muli-line string. # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector # Example: - # nodeSelector: | - # beta.kubernetes.io/arch: amd64 nodeSelector: | - sw-backend: "true" + beta.kubernetes.io/arch: amd64 + # nodeSelector: | + # sw-backend: "true" # used to assign priority to server pods # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ @@ -389,7 +389,7 @@ filer: # extraEnvVars is a list of extra enviroment variables to set with the stateful set. extraEnvironmentVars: - WEED_MYSQL_ENABLED: "true" + WEED_MYSQL_ENABLED: "false" WEED_MYSQL_HOSTNAME: "mysql-db-host" WEED_MYSQL_PORT: "3306" WEED_MYSQL_DATABASE: "sw_database" @@ -400,7 +400,7 @@ filer: # enable usage of memsql as filer backend WEED_MYSQL_INTERPOLATEPARAMS: "true" # if you want to use leveldb2, then should enable "enablePVC". or you may lose your data. - WEED_LEVELDB2_ENABLED: "false" + WEED_LEVELDB2_ENABLED: "true" # with http DELETE, by default the filer would check whether a folder is empty. # recursive_delete will delete all sub folders and files, similar to "rm -Rf" WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false" @@ -410,7 +410,7 @@ filer: s3: enabled: true port: 8333 - #allow empty folders + # allow empty folders allowEmptyFolder: false # Suffix of the host name, {bucket}.{domainName} domainName: "" @@ -430,7 +430,7 @@ s3: port: 8333 metricsPort: 9327 loggingOverrideLevel: null - #allow empty folders + # allow empty folders allowEmptyFolder: true # enable user & permission to s3 (need to inject to all services) enableAuth: false @@ -449,7 +449,7 @@ s3: # should map directly to the value of the resources field for a PodSpec, # formatted as a multi-line string. By default no direct resource request # is made. - resources: null + resources: {} # Toleration Settings for server pods # This should be a multi-line string matching the Toleration array @@ -459,10 +459,10 @@ s3: # nodeSelector labels for server pod assignment, formatted as a muli-line string. # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector # Example: - # nodeSelector: | - # beta.kubernetes.io/arch: amd64 nodeSelector: | - sw-backend: "true" + beta.kubernetes.io/arch: amd64 + # nodeSelector: | + # sw-backend: "true" # used to assign priority to server pods # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ @@ -479,5 +479,5 @@ certificates: ipAddresses: [] keyAlgorithm: rsa keySize: 2048 - duration: 2160h # 90d - renewBefore: 360h # 15d + duration: 2160h # 90d + renewBefore: 360h # 15d