From a522507f9566b4ff2f98860830693bc660b4353c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Fri, 7 Oct 2022 16:20:34 +0500 Subject: [PATCH] configure raft metrics (#3798) --- weed/server/raft_hashicorp.go | 16 ++++++++++++++ weed/stats/metrics.go | 41 ++++++++++++++++++----------------- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/weed/server/raft_hashicorp.go b/weed/server/raft_hashicorp.go index 8f81924d1..76468b5e1 100644 --- a/weed/server/raft_hashicorp.go +++ b/weed/server/raft_hashicorp.go @@ -6,10 +6,13 @@ package weed_server import ( "fmt" transport "github.com/Jille/raft-grpc-transport" + "github.com/armon/go-metrics" + "github.com/armon/go-metrics/prometheus" "github.com/hashicorp/raft" boltdb "github.com/hashicorp/raft-boltdb/v2" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/stats" "google.golang.org/grpc" "math/rand" "os" @@ -182,5 +185,18 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { }() } + // Configure a prometheus sink as the raft metrics sink + if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{ + Registerer: stats.Gather, + }); err != nil { + return nil, fmt.Errorf("NewPrometheusSink: %v", err) + } else { + metricsConf := metrics.DefaultConfig(stats.Namespace) + metricsConf.EnableRuntimeMetrics = false + if _, err = metrics.NewGlobal(metricsConf, sink); err != nil { + return nil, fmt.Errorf("metrics.NewGlobal: %v", err) + } + } + return s, nil } diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 38c5590b3..9947e2316 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -19,6 +19,7 @@ import ( // Readonly volume types const ( + Namespace = "SeaweedFS" IsReadOnly = "IsReadOnly" NoWriteOrDelete = "noWriteOrDelete" NoWriteCanDelete = "noWriteCanDelete" @@ -32,7 +33,7 @@ var ( MasterClientConnectCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "wdclient", Name: "connect_updates", Help: "Counter of master client leader updates.", @@ -40,7 +41,7 @@ var ( MasterRaftIsleader = prometheus.NewGauge( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "master", Name: "is_leader", Help: "is leader", @@ -48,7 +49,7 @@ var ( MasterReceivedHeartbeatCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "master", Name: "received_heartbeats", Help: "Counter of master received heartbeat.", @@ -56,7 +57,7 @@ var ( MasterReplicaPlacementMismatch = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "master", Name: "replica_placement_mismatch", Help: "replica placement mismatch", @@ -64,7 +65,7 @@ var ( MasterLeaderChangeCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "master", Name: "leader_changes", Help: "Counter of master leader changes.", @@ -72,7 +73,7 @@ var ( FilerRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "filer", Name: "request_total", Help: "Counter of filer requests.", @@ -80,7 +81,7 @@ var ( FilerRequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "filer", Name: "request_seconds", Help: "Bucketed histogram of filer request processing time.", @@ -89,7 +90,7 @@ var ( FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "filer", Name: "last_send_timestamp_of_subscribe", Help: "The last send timestamp of the filer subscription.", @@ -97,7 +98,7 @@ var ( FilerStoreCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "filerStore", Name: "request_total", Help: "Counter of filer store requests.", @@ -105,7 +106,7 @@ var ( FilerStoreHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "filerStore", Name: "request_seconds", Help: "Bucketed histogram of filer store request processing time.", @@ -114,7 +115,7 @@ var ( FilerSyncOffsetGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "filerSync", Name: "sync_offset", Help: "The offset of the filer synchronization service.", @@ -122,7 +123,7 @@ var ( VolumeServerRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "volumeServer", Name: "request_total", Help: "Counter of volume server requests.", @@ -130,7 +131,7 @@ var ( VolumeServerRequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "volumeServer", Name: "request_seconds", Help: "Bucketed histogram of volume server request processing time.", @@ -139,7 +140,7 @@ var ( VolumeServerVolumeCounter = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "volumeServer", Name: "volumes", Help: "Number of volumes or shards.", @@ -147,7 +148,7 @@ var ( VolumeServerReadOnlyVolumeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "volumeServer", Name: "read_only_volumes", Help: "Number of read only volumes.", @@ -155,7 +156,7 @@ var ( VolumeServerMaxVolumeCounter = prometheus.NewGauge( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "volumeServer", Name: "max_volumes", Help: "Maximum number of volumes.", @@ -163,7 +164,7 @@ var ( VolumeServerDiskSizeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "volumeServer", Name: "total_disk_size", Help: "Actual disk size used by volumes.", @@ -171,7 +172,7 @@ var ( VolumeServerResourceGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "volumeServer", Name: "resource", Help: "Resource usage", @@ -179,7 +180,7 @@ var ( S3RequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "s3", Name: "request_total", Help: "Counter of s3 requests.", @@ -187,7 +188,7 @@ var ( S3RequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "SeaweedFS", + Namespace: Namespace, Subsystem: "s3", Name: "request_seconds", Help: "Bucketed histogram of s3 request processing time.",