metric shows who is currently blocking the cluster or not (#3799)

* master_admin_lock Shows whether cluster is locked now or not
https://github.com/seaweedfs/seaweedfs/issues/3452

* fix metric MasterAdminLock

weed/server/master_grpc_server.go

@@ -307,6 +307,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 		case <-ticker.C:
 			if !ms.Topo.IsLeader() {
 				stats.MasterRaftIsleader.Set(0)
+				stats.MasterAdminLock.Reset()
+				stats.MasterReplicaPlacementMismatch.Reset()
 				return ms.informNewLeader(stream)
 			} else {
 				stats.MasterRaftIsleader.Set(1)

weed/server/master_grpc_server_admin.go

@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"math/rand"
 	"sync"
 	"time"
@@ -112,11 +113,13 @@ func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts t
 		lastClient:     clientName,
 	}
 	locks.locks[lockName] = lock
+	stats.MasterAdminLock.WithLabelValues(clientName).Set(1)
 	return lock.accessLockTime, lock.accessSecret
 }
 
 func (locks *AdminLocks) deleteLock(lockName string) {
 	locks.Lock()
+	stats.MasterAdminLock.WithLabelValues(locks.locks[lockName].lastClient).Set(0)
 	defer locks.Unlock()
 	delete(locks.locks, lockName)
 }

weed/stats/metrics.go

@@ -47,6 +47,14 @@ var (
 			Help:      "is leader",
 		})
 
+	MasterAdminLock = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: "master",
+			Name:      "admin_lock",
+			Help:      "admin lock",
+		}, []string{"client"})
+
 	MasterReceivedHeartbeatCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -199,6 +207,7 @@ var (
 func init() {
 	Gather.MustRegister(MasterClientConnectCounter)
 	Gather.MustRegister(MasterRaftIsleader)
+	Gather.MustRegister(MasterAdminLock)
 	Gather.MustRegister(MasterReceivedHeartbeatCounter)
 	Gather.MustRegister(MasterLeaderChangeCounter)
 	Gather.MustRegister(MasterReplicaPlacementMismatch)