Helm chart monitoring improvements (#3406)

k8s/helm_charts2/dashboards/seaweedfs-grafana-dashboard.json

@@ -1,14 +1,5 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_PROMETHEUS-DEV",
-      "label": "prometheus-dev",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
-  ],
+  "__inputs": [],
   "__requires": [
     {
       "type": "grafana",
@@ -33,7 +24,7 @@
     "list": [
       {
         "builtIn": 1,
-        "datasource": "${DS_PROMETHEUS-DEV}",
+        "datasource": "${DS_PROMETHEUS}",
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
@@ -61,7 +52,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -151,7 +142,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -247,7 +238,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -343,7 +334,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 0,
@@ -448,7 +439,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -538,7 +529,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -628,7 +619,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -718,7 +709,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 0,
@@ -811,7 +802,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 0,
@@ -943,7 +934,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -1031,7 +1022,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -1117,7 +1108,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "fill": 1,
           "id": 48,
           "legend": {
@@ -1200,7 +1191,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "fill": 1,
           "id": 50,
           "legend": {
@@ -1283,7 +1274,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "fill": 1,
           "id": 51,
           "legend": {
@@ -1371,7 +1362,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -1451,7 +1442,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -1546,7 +1537,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -1651,7 +1642,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -1731,7 +1722,7 @@
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
           "editable": true,
           "error": false,
           "fill": 1,
@@ -1819,7 +1810,29 @@
   "style": "dark",
   "tags": [],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
   },
   "time": {
     "from": "now-30d",

k8s/helm_charts2/templates/filer-service.yaml

@@ -20,6 +20,12 @@ spec:
     port: {{ .Values.filer.grpcPort }}
     targetPort: {{ .Values.filer.grpcPort }}
     protocol: TCP
+  {{- if .Values.filer.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.filer.metricsPort }}
+    targetPort: {{ .Values.filer.metricsPort }}
+    protocol: TCP
+  {{- end }}
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: filer

k8s/helm_charts2/templates/filer-servicemonitor.yaml

@@ -11,10 +11,11 @@ metadata:
 spec:
   endpoints:
     - interval: 30s
-      port: swfs-filer-metrics
+      port: metrics
       scrapeTimeout: 5s
   selector:
-    app: {{ template "seaweedfs.name" . }}
-    component: filer
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: filer
 {{- end }}
 {{- end }}

k8s/helm_charts2/templates/filer-statefulset.yaml

@@ -110,7 +110,7 @@ spec:
               filer \
               -port={{ .Values.filer.port }} \
               {{- if .Values.filer.metricsPort }}
-              -metricsPort {{ .Values.filer.metricsPort }} \
+              -metricsPort={{ .Values.filer.metricsPort }} \
               {{- end }}
               {{- if .Values.filer.redirectOnRead }}
               -redirectOnRead \
@@ -193,6 +193,8 @@ spec:
           ports:
             - containerPort: {{ .Values.filer.port }}
               name: swfs-filer
+            - containerPort: {{ .Values.filer.metricsPort }}
+              name: metrics
             - containerPort: {{ .Values.filer.grpcPort }}
               #name: swfs-filer-grpc
           readinessProbe:

k8s/helm_charts2/templates/master-service.yaml

@@ -20,6 +20,12 @@ spec:
     port: {{ .Values.master.grpcPort }}
     targetPort: {{ .Values.master.grpcPort }}
     protocol: TCP
+  {{- if .Values.master.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.master.metricsPort }}
+    targetPort: {{ .Values.master.metricsPort }}
+    protocol: TCP
+  {{- end }}
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: master

k8s/helm_charts2/templates/master-servicemonitor.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.master.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-master
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: master
+spec:
+  endpoints:
+    - interval: 30s
+      port: metrics
+      scrapeTimeout: 5s
+  selector:
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: master
+{{- end }}
+{{- end }}

k8s/helm_charts2/templates/master-statefulset.yaml

@@ -109,7 +109,14 @@ spec:
               -volumePreallocate \
               {{- end }}
               {{- if .Values.global.monitoring.enabled }}
+              {{- if and .Values.global.monitoring.gatewayHost .Values.global.monitoring.gatewayPort }}
               -metrics.address="{{ .Values.global.monitoring.gatewayHost }}:{{ .Values.global.monitoring.gatewayPort }}" \
+              {{- if .Values.master.metricsIntervalSec }}
+              -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
+              {{- end }}
+              {{- else if .Values.master.metricsPort }}
+              -metricsPort={{ .Values.master.metricsPort }} \
+              {{- end }}
               {{- end }}
               -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
               {{- if .Values.master.disableHttp }}
@@ -121,9 +128,6 @@ spec:
               {{- if .Values.master.garbageThreshold }}
               -garbageThreshold={{ .Values.master.garbageThreshold }} \
               {{- end }}
-              {{- if .Values.master.metricsIntervalSec }}
-              -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
-              {{- end }}
               -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
               -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
           volumeMounts:
@@ -156,6 +160,10 @@ spec:
           ports:
             - containerPort: {{ .Values.master.port }}
               name: swfs-master
+            {{- if and .Values.global.monitoring.enabled .Values.master.metricsPort }}
+            - containerPort: {{ .Values.master.metricsPort }}
+              name: metrics
+            {{- end }}
             - containerPort: {{ .Values.master.grpcPort }}
               #name: swfs-master-grpc
           readinessProbe:

k8s/helm_charts2/templates/s3-deployment.yaml

@@ -131,6 +131,10 @@ spec:
           ports:
             - containerPort: {{ .Values.s3.port }}
               name: swfs-s3
+            {{- if .Values.s3.metricsPort }}
+            - containerPort: {{ .Values.s3.metricsPort }}
+              name: "metrics"
+            {{- end }}
           readinessProbe:
             httpGet:
               path: /status

k8s/helm_charts2/templates/s3-servicemonitor.yaml

@@ -11,10 +11,11 @@ metadata:
 spec:
   endpoints:
     - interval: 30s
-      port: swfs-s3-metrics
+      port: metrics
       scrapeTimeout: 5s
   selector:
-    app: {{ template "seaweedfs.name" . }}
-    component: s3
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: s3
 {{- end }}
 {{- end }}

k8s/helm_charts2/templates/volume-service.yaml

@@ -18,7 +18,7 @@ spec:
     targetPort: {{ .Values.volume.grpcPort }}
     protocol: TCP
 {{- if .Values.volume.metricsPort }}
-  - name: "swfs-volume-metrics"
+  - name: "metrics"
     port: {{ .Values.volume.metricsPort }}
     targetPort: {{ .Values.volume.metricsPort }}
     protocol: TCP

k8s/helm_charts2/templates/volume-servicemonitor.yaml

@@ -11,10 +11,11 @@ metadata:
 spec:
   endpoints:
     - interval: 30s
-      port: swfs-volume-metrics
+      port: metrics
       scrapeTimeout: 5s
   selector:
-    app: {{ template "seaweedfs.name" . }}
-    component: volume
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: volume
 {{- end }}
 {{- end }}

k8s/helm_charts2/templates/volume-statefulset.yaml

@@ -102,7 +102,7 @@ spec:
                 volume \
                 -port={{ .Values.volume.port }} \
                 {{- if .Values.volume.metricsPort }}
-                -metricsPort {{ .Values.volume.metricsPort }} \
+                -metricsPort={{ .Values.volume.metricsPort }} \
                 {{- end }}
                 -dir={{ .Values.volume.dir }} \
                 {{- if .Values.volume.dir_idx }}
@@ -170,8 +170,12 @@ spec:
           ports:
             - containerPort: {{ .Values.volume.port }}
               name: swfs-vol
+            {{- if .Values.volume.metricsPort }}
+            - containerPort: {{ .Values.filer.metricsPort }}
+              name: metrics
+            {{- end }}
             - containerPort: {{ .Values.volume.grpcPort }}
-              #name: swfs-vol-grpc
+              name: swfs-vol-grpc
           readinessProbe:
             httpGet:
               path: /status

k8s/helm_charts2/values.yaml

@@ -39,6 +39,7 @@ master:
   replicas: 1
   port: 9333
   grpcPort: 19333
+  metricsPort: 9327
   ipBind: "0.0.0.0"
   volumePreallocate: false
   volumeSizeLimitMB: 1000

other/metrics/grafana_seaweedfs_k8s.json

@@ -1891,7 +1891,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -1909,7 +1909,7 @@
           "refId": "A"
         },
         {
-          "expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
           "format": "time_series",
           "hide": true,
           "interval": "",
@@ -1918,7 +1918,7 @@
           "refId": "C"
         },
         {
-          "expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
           "format": "time_series",
           "hide": true,
           "interval": "",
@@ -2019,7 +2019,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"metrics\"}",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -2119,7 +2119,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"metrics\"}",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -2203,7 +2203,7 @@
           "value": "s3"
         },
         "datasource": "$DS_PROMETHEUS",
-        "definition": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)",
+        "definition": "label_values({endpoint=\"metrics\"}, namespace)",
         "hide": 0,
         "includeAll": false,
         "label": "namespace",
@@ -2216,7 +2216,7 @@
             "value": "s3"
           }
         ],
-        "query": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)",
+        "query": "label_values({endpoint=\"metrics\"}, namespace)",
         "refresh": 0,
         "regex": "",
         "skipUrlSync": false,