Helm chart monitoring improvements (#3406)

2 years ago · 28a1f42962
14 changed files with 122 additions and 54 deletions
--- a/k8s/helm_charts2/dashboards/seaweedfs-grafana-dashboard.json
+++ b/k8s/helm_charts2/dashboards/seaweedfs-grafana-dashboard.json
@ -1,14 +1,5 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_PROMETHEUS-DEV",
-      "label": "prometheus-dev",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
-  ],
+  "__inputs": [],
  "__requires": [
    {
      "type": "grafana",
@ -33,7 +24,7 @@
    "list": [
      {
        "builtIn": 1,
-        "datasource": "${DS_PROMETHEUS-DEV}",
+        "datasource": "${DS_PROMETHEUS}",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
@ -61,7 +52,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -151,7 +142,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -247,7 +238,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -343,7 +334,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 0,
@ -448,7 +439,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -538,7 +529,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -628,7 +619,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -718,7 +709,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 0,
@ -811,7 +802,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 0,
@ -943,7 +934,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -1031,7 +1022,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -1117,7 +1108,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "id": 48,
          "legend": {
@ -1200,7 +1191,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "id": 50,
          "legend": {
@ -1283,7 +1274,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "fill": 1,
          "id": 51,
          "legend": {
@ -1371,7 +1362,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -1451,7 +1442,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -1546,7 +1537,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -1651,7 +1642,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -1731,7 +1722,7 @@
          "bars": false,
          "dashLength": 10,
          "dashes": false,
-          "datasource": "${DS_PROMETHEUS-DEV}",
+          "datasource": "${DS_PROMETHEUS}",
          "editable": true,
          "error": false,
          "fill": 1,
@ -1819,7 +1810,29 @@
  "style": "dark",
  "tags": [],
  "templating": {
-    "list": []
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
  },
  "time": {
    "from": "now-30d",
--- a/k8s/helm_charts2/templates/filer-service.yaml
+++ b/k8s/helm_charts2/templates/filer-service.yaml
@ -20,6 +20,12 @@ spec:
    port: {{ .Values.filer.grpcPort }}
    targetPort: {{ .Values.filer.grpcPort }}
    protocol: TCP
+  {{- if .Values.filer.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.filer.metricsPort }}
+    targetPort: {{ .Values.filer.metricsPort }}
+    protocol: TCP
+  {{- end }}
  selector:
    app: {{ template "seaweedfs.name" . }}
    component: filer
--- a/k8s/helm_charts2/templates/filer-servicemonitor.yaml
+++ b/k8s/helm_charts2/templates/filer-servicemonitor.yaml
@ -11,10 +11,11 @@ metadata:
 spec:
  endpoints:
    - interval: 30s
-      port: swfs-filer-metrics
+      port: metrics
      scrapeTimeout: 5s
  selector:
-    app: {{ template "seaweedfs.name" . }}
-    component: filer
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: filer
 {{- end }}
 {{- end }}
--- a/k8s/helm_charts2/templates/filer-statefulset.yaml
+++ b/k8s/helm_charts2/templates/filer-statefulset.yaml
@ -110,7 +110,7 @@ spec:
              filer \
              -port={{ .Values.filer.port }} \
              {{- if .Values.filer.metricsPort }}
-              -metricsPort {{ .Values.filer.metricsPort }} \
+              -metricsPort={{ .Values.filer.metricsPort }} \
              {{- end }}
              {{- if .Values.filer.redirectOnRead }}
              -redirectOnRead \
@ -193,6 +193,8 @@ spec:
          ports:
            - containerPort: {{ .Values.filer.port }}
              name: swfs-filer
+            - containerPort: {{ .Values.filer.metricsPort }}
+              name: metrics
            - containerPort: {{ .Values.filer.grpcPort }}
              #name: swfs-filer-grpc
          readinessProbe:
--- a/k8s/helm_charts2/templates/master-service.yaml
+++ b/k8s/helm_charts2/templates/master-service.yaml
@ -20,6 +20,12 @@ spec:
    port: {{ .Values.master.grpcPort }}
    targetPort: {{ .Values.master.grpcPort }}
    protocol: TCP
+  {{- if .Values.master.metricsPort }}
+  - name: "metrics"
+    port: {{ .Values.master.metricsPort }}
+    targetPort: {{ .Values.master.metricsPort }}
+    protocol: TCP
+  {{- end }}
  selector:
    app: {{ template "seaweedfs.name" . }}
    component: master
--- a/k8s/helm_charts2/templates/master-servicemonitor.yaml
+++ b/k8s/helm_charts2/templates/master-servicemonitor.yaml
@ -0,0 +1,21 @@
+{{- if .Values.master.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "seaweedfs.name" . }}-master
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: master
+spec:
+  endpoints:
+    - interval: 30s
+      port: metrics
+      scrapeTimeout: 5s
+  selector:
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: master
+{{- end }}
+{{- end }}
--- a/k8s/helm_charts2/templates/master-statefulset.yaml
+++ b/k8s/helm_charts2/templates/master-statefulset.yaml
@ -109,7 +109,14 @@ spec:
              -volumePreallocate \
              {{- end }}
              {{- if .Values.global.monitoring.enabled }}
+              {{- if and .Values.global.monitoring.gatewayHost .Values.global.monitoring.gatewayPort }}
              -metrics.address="{{ .Values.global.monitoring.gatewayHost }}:{{ .Values.global.monitoring.gatewayPort }}" \
+              {{- if .Values.master.metricsIntervalSec }}
+              -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
+              {{- end }}
+              {{- else if .Values.master.metricsPort }}
+              -metricsPort={{ .Values.master.metricsPort }} \
+              {{- end }}
              {{- end }}
              -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
              {{- if .Values.master.disableHttp }}
@ -121,9 +128,6 @@ spec:
              {{- if .Values.master.garbageThreshold }}
              -garbageThreshold={{ .Values.master.garbageThreshold }} \
              {{- end }}
-              {{- if .Values.master.metricsIntervalSec }}
-              -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
-              {{- end }}
              -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
          volumeMounts:
@ -156,6 +160,10 @@ spec:
          ports:
            - containerPort: {{ .Values.master.port }}
              name: swfs-master
+            {{- if and .Values.global.monitoring.enabled .Values.master.metricsPort }}
+            - containerPort: {{ .Values.master.metricsPort }}
+              name: metrics
+            {{- end }}
            - containerPort: {{ .Values.master.grpcPort }}
              #name: swfs-master-grpc
          readinessProbe:
--- a/k8s/helm_charts2/templates/s3-deployment.yaml
+++ b/k8s/helm_charts2/templates/s3-deployment.yaml
@ -131,6 +131,10 @@ spec:
          ports:
            - containerPort: {{ .Values.s3.port }}
              name: swfs-s3
+            {{- if .Values.s3.metricsPort }}
+            - containerPort: {{ .Values.s3.metricsPort }}
+              name: "metrics"
+            {{- end }}
          readinessProbe:
            httpGet:
              path: /status
--- a/k8s/helm_charts2/templates/s3-servicemonitor.yaml
+++ b/k8s/helm_charts2/templates/s3-servicemonitor.yaml
@ -11,10 +11,11 @@ metadata:
 spec:
  endpoints:
    - interval: 30s
-      port: swfs-s3-metrics
+      port: metrics
      scrapeTimeout: 5s
  selector:
-    app: {{ template "seaweedfs.name" . }}
-    component: s3
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: s3
 {{- end }}
 {{- end }}
--- a/k8s/helm_charts2/templates/volume-service.yaml
+++ b/k8s/helm_charts2/templates/volume-service.yaml
@ -18,7 +18,7 @@ spec:
    targetPort: {{ .Values.volume.grpcPort }}
    protocol: TCP
 {{- if .Values.volume.metricsPort }}
-  - name: "swfs-volume-metrics"
+  - name: "metrics"
    port: {{ .Values.volume.metricsPort }}
    targetPort: {{ .Values.volume.metricsPort }}
    protocol: TCP
--- a/k8s/helm_charts2/templates/volume-servicemonitor.yaml
+++ b/k8s/helm_charts2/templates/volume-servicemonitor.yaml
@ -11,10 +11,11 @@ metadata:
 spec:
  endpoints:
    - interval: 30s
-      port: swfs-volume-metrics
+      port: metrics
      scrapeTimeout: 5s
  selector:
-    app: {{ template "seaweedfs.name" . }}
-    component: volume
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      component: volume
 {{- end }}
 {{- end }}
--- a/k8s/helm_charts2/templates/volume-statefulset.yaml
+++ b/k8s/helm_charts2/templates/volume-statefulset.yaml
@ -102,7 +102,7 @@ spec:
                volume \
                -port={{ .Values.volume.port }} \
                {{- if .Values.volume.metricsPort }}
-                -metricsPort {{ .Values.volume.metricsPort }} \
+                -metricsPort={{ .Values.volume.metricsPort }} \
                {{- end }}
                -dir={{ .Values.volume.dir }} \
                {{- if .Values.volume.dir_idx }}
@ -170,8 +170,12 @@ spec:
          ports:
            - containerPort: {{ .Values.volume.port }}
              name: swfs-vol
+            {{- if .Values.volume.metricsPort }}
+            - containerPort: {{ .Values.filer.metricsPort }}
+              name: metrics
+            {{- end }}
            - containerPort: {{ .Values.volume.grpcPort }}
-              #name: swfs-vol-grpc
+              name: swfs-vol-grpc
          readinessProbe:
            httpGet:
              path: /status
--- a/k8s/helm_charts2/values.yaml
+++ b/k8s/helm_charts2/values.yaml
@ -39,6 +39,7 @@ master:
  replicas: 1
  port: 9333
  grpcPort: 19333
+  metricsPort: 9327
  ipBind: "0.0.0.0"
  volumePreallocate: false
  volumeSizeLimitMB: 1000
--- a/other/metrics/grafana_seaweedfs_k8s.json
+++ b/other/metrics/grafana_seaweedfs_k8s.json
@ -1891,7 +1891,7 @@
      "steppedLine": false,
      "targets": [
        {
-          "expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
          "format": "time_series",
          "hide": false,
          "interval": "",
@ -1909,7 +1909,7 @@
          "refId": "A"
        },
        {
-          "expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
          "format": "time_series",
          "hide": true,
          "interval": "",
@ -1918,7 +1918,7 @@
          "refId": "C"
        },
        {
-          "expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
          "format": "time_series",
          "hide": true,
          "interval": "",
@ -2019,7 +2019,7 @@
      "steppedLine": false,
      "targets": [
        {
-          "expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"metrics\"}",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
@ -2119,7 +2119,7 @@
      "steppedLine": false,
      "targets": [
        {
-          "expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
+          "expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"metrics\"}",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
@ -2203,7 +2203,7 @@
          "value": "s3"
        },
        "datasource": "$DS_PROMETHEUS",
-        "definition": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)",
+        "definition": "label_values({endpoint=\"metrics\"}, namespace)",
        "hide": 0,
        "includeAll": false,
        "label": "namespace",
@ -2216,7 +2216,7 @@
            "value": "s3"
          }
        ],
-        "query": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)",
+        "query": "label_values({endpoint=\"metrics\"}, namespace)",
        "refresh": 0,
        "regex": "",
        "skipUrlSync": false,