Merge pull request #3163 from kmlebedev/metric_replica_placement_mismatch

stats master_replica_placement_mismatch

docker/prometheus/prometheus.yml

@@ -8,6 +8,7 @@ scrape_configs:
     static_configs:
       - targets:
           - 'prometheus:9090'
+          - 'master:9324'
           - 'volume:9325'
           - 'filer:9326'
           - 's3:9327'

docker/seaweedfs-compose.yml

@@ -6,7 +6,8 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "master -ip=master -ip.bind=0.0.0.0"
+      - 9324:9324
+    command: "master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324"
   volume:
     image: chrislusf/seaweedfs # use a remote image
     ports:

weed/stats/metrics.go

@@ -44,6 +44,14 @@ var (
 			Help:      "Counter of master received heartbeat.",
 		}, []string{"type"})
 
+	MasterReplicaPlacementMismatch = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "master",
+			Name:      "replica_placement_mismatch",
+			Help:      "replica placement mismatch",
+		}, []string{"collection", "id"})
+
 	MasterLeaderChangeCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "SeaweedFS",
@@ -165,6 +173,7 @@ func init() {
 	Gather.MustRegister(MasterRaftIsleader)
 	Gather.MustRegister(MasterReceivedHeartbeatCounter)
 	Gather.MustRegister(MasterLeaderChangeCounter)
+	Gather.MustRegister(MasterReplicaPlacementMismatch)
 
 	Gather.MustRegister(FilerRequestCounter)
 	Gather.MustRegister(FilerRequestHistogram)

weed/topology/node.go

@@ -3,6 +3,7 @@ package topology
 import (
 	"errors"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
@@ -246,6 +247,14 @@ func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSi
 				} else if float64(v.Size) > float64(volumeSizeLimit)*growThreshold {
 					n.GetTopology().chanCrowdedVolumes <- v
 				}
+				copyCount := v.ReplicaPlacement.GetCopyCount()
+				if copyCount > 1 {
+					if copyCount > len(n.GetTopology().Lookup(v.Collection, v.Id)) {
+						stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
+					} else {
+						stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0)
+					}
+				}
 			}
 		}
 	} else {