From fb57aa431c52b7b5575e2eadd52ed3045d6c6c51 Mon Sep 17 00:00:00 2001
From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com>
Date: Fri, 10 Jun 2022 15:30:40 +0500
Subject: [PATCH] stats master_replica_placement_mismatch

---
 docker/prometheus/prometheus.yml | 1 +
 docker/seaweedfs-compose.yml     | 3 ++-
 weed/stats/metrics.go            | 9 +++++++++
 weed/topology/node.go            | 9 +++++++++
 4 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml
index 34f669d56..2f0ead441 100644
--- a/docker/prometheus/prometheus.yml
+++ b/docker/prometheus/prometheus.yml
@@ -8,6 +8,7 @@ scrape_configs:
     static_configs:
       - targets:
           - 'prometheus:9090'
+          - 'master:9324'
           - 'volume:9325'
           - 'filer:9326'
           - 's3:9327'
\ No newline at end of file
diff --git a/docker/seaweedfs-compose.yml b/docker/seaweedfs-compose.yml
index 706f10180..cce1c39ef 100644
--- a/docker/seaweedfs-compose.yml
+++ b/docker/seaweedfs-compose.yml
@@ -6,7 +6,8 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "master -ip=master -ip.bind=0.0.0.0"
+      - 9324:9324
+    command: "master -ip=master -ip.bind=0.0.0.0 -metricsPort=9324"
   volume:
     image: chrislusf/seaweedfs # use a remote image
     ports:
diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go
index 8381999c2..943aafff9 100644
--- a/weed/stats/metrics.go
+++ b/weed/stats/metrics.go
@@ -44,6 +44,14 @@ var (
 			Help:      "Counter of master received heartbeat.",
 		}, []string{"type"})
 
+	MasterReplicaPlacementMismatch = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "SeaweedFS",
+			Subsystem: "master",
+			Name:      "replica_placement_mismatch",
+			Help:      "replica placement mismatch",
+		}, []string{"collection", "id"})
+
 	MasterLeaderChangeCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "SeaweedFS",
@@ -165,6 +173,7 @@ func init() {
 	Gather.MustRegister(MasterRaftIsleader)
 	Gather.MustRegister(MasterReceivedHeartbeatCounter)
 	Gather.MustRegister(MasterLeaderChangeCounter)
+	Gather.MustRegister(MasterReplicaPlacementMismatch)
 
 	Gather.MustRegister(FilerRequestCounter)
 	Gather.MustRegister(FilerRequestHistogram)
diff --git a/weed/topology/node.go b/weed/topology/node.go
index 4772cb411..c5956177a 100644
--- a/weed/topology/node.go
+++ b/weed/topology/node.go
@@ -3,6 +3,7 @@ package topology
 import (
 	"errors"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
@@ -246,6 +247,14 @@ func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSi
 				} else if float64(v.Size) > float64(volumeSizeLimit)*growThreshold {
 					n.GetTopology().chanCrowdedVolumes <- v
 				}
+				copyCount := v.ReplicaPlacement.GetCopyCount()
+				if copyCount > 1 {
+					if copyCount > len(n.GetTopology().Lookup(v.Collection, v.Id)) {
+						stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
+					} else {
+						stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0)
+					}
+				}
 			}
 		}
 	} else {