From 709c83716cbc1877510d0836ab1194eb28c99761 Mon Sep 17 00:00:00 2001 From: Evgeny Kuzhelev Date: Wed, 27 Jul 2022 13:55:09 +0500 Subject: [PATCH 1/3] delete disk_size metrics when collection deleted --- weed/storage/store.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/weed/storage/store.go b/weed/storage/store.go index f833f1b15..8ab8b994c 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -101,6 +101,7 @@ func (s *Store) DeleteCollection(collection string) (e error) { if e != nil { return } + stats.VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal") // let the heartbeat send the list of volumes, instead of sending the deleted volume ids to DeletedVolumesChan } return @@ -240,19 +241,19 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { if maxFileKey < curMaxFileKey { maxFileKey = curMaxFileKey } - deleteVolume := false + shouldDeleteVolume := false if !v.expired(volumeMessage.Size, s.GetVolumeSizeLimit()) { volumeMessages = append(volumeMessages, volumeMessage) } else { if v.expiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) { deleteVids = append(deleteVids, v.Id) - deleteVolume = true + shouldDeleteVolume = true } else { glog.V(0).Infof("volume %d is expired", v.Id) } if v.lastIoError != nil { deleteVids = append(deleteVids, v.Id) - deleteVolume = true + shouldDeleteVolume = true glog.Warningf("volume %d has IO error: %v", v.Id, v.lastIoError) } } @@ -260,12 +261,12 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { if _, exist := collectionVolumeSize[v.Collection]; !exist { collectionVolumeSize[v.Collection] = 0 } - if !deleteVolume { + if !shouldDeleteVolume { collectionVolumeSize[v.Collection] += volumeMessage.Size } else { collectionVolumeSize[v.Collection] -= volumeMessage.Size - if collectionVolumeSize[v.Collection] <= 0 { - delete(collectionVolumeSize, v.Collection) + if collectionVolumeSize[v.Collection] < 0 { + collectionVolumeSize[v.Collection] = 0 } } @@ -277,7 +278,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { "isDiskSpaceLow": 0, } } - if !deleteVolume && v.IsReadOnly() { + if !shouldDeleteVolume && v.IsReadOnly() { collectionVolumeReadOnlyCount[v.Collection]["IsReadOnly"] += 1 if v.noWriteOrDelete { collectionVolumeReadOnlyCount[v.Collection]["noWriteOrDelete"] += 1 From 47c72e6f3562bdf8566effd38c911285ba955eb4 Mon Sep 17 00:00:00 2001 From: Evgeny Kuzhelev Date: Wed, 27 Jul 2022 16:31:49 +0500 Subject: [PATCH 2/3] remove all (currently existing) collection volume metrics --- weed/stats/metrics.go | 18 ++++++++++++++++++ weed/storage/store.go | 19 ++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index f0b810608..0f4fcfb52 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -17,6 +17,16 @@ import ( "github.com/prometheus/client_golang/prometheus/push" ) +// Readonly volume types +const ( + IsReadOnly = "IsReadOnly" + NoWriteOrDelete = "noWriteOrDelete" + NoWriteCanDelete = "noWriteCanDelete" + IsDiskSpaceLow = "isDiskSpaceLow" +) + +var readOnlyVolumeTypes = [4]string{IsReadOnly, NoWriteOrDelete, NoWriteCanDelete, IsDiskSpaceLow} + var ( Gather = prometheus.NewRegistry() @@ -249,3 +259,11 @@ func SourceName(port uint32) string { } return net.JoinHostPort(hostname, strconv.Itoa(int(port))) } + +func DeleteCollectionMetrics(collection string) { + VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal") + for _, volume_type := range readOnlyVolumeTypes { + VolumeServerReadOnlyVolumeGauge.DeleteLabelValues(collection, volume_type) + } + VolumeServerVolumeCounter.DeleteLabelValues(collection, "volume") +} diff --git a/weed/storage/store.go b/weed/storage/store.go index 8ab8b994c..06c6362ba 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -101,7 +101,7 @@ func (s *Store) DeleteCollection(collection string) (e error) { if e != nil { return } - stats.VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal") + stats.DeleteCollectionMetrics(collection) // let the heartbeat send the list of volumes, instead of sending the deleted volume ids to DeletedVolumesChan } return @@ -272,22 +272,22 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { if _, exist := collectionVolumeReadOnlyCount[v.Collection]; !exist { collectionVolumeReadOnlyCount[v.Collection] = map[string]uint8{ - "IsReadOnly": 0, - "noWriteOrDelete": 0, - "noWriteCanDelete": 0, - "isDiskSpaceLow": 0, + stats.IsReadOnly: 0, + stats.NoWriteOrDelete: 0, + stats.NoWriteCanDelete: 0, + stats.IsDiskSpaceLow: 0, } } if !shouldDeleteVolume && v.IsReadOnly() { - collectionVolumeReadOnlyCount[v.Collection]["IsReadOnly"] += 1 + collectionVolumeReadOnlyCount[v.Collection][stats.IsReadOnly] += 1 if v.noWriteOrDelete { - collectionVolumeReadOnlyCount[v.Collection]["noWriteOrDelete"] += 1 + collectionVolumeReadOnlyCount[v.Collection][stats.NoWriteOrDelete] += 1 } if v.noWriteCanDelete { - collectionVolumeReadOnlyCount[v.Collection]["noWriteCanDelete"] += 1 + collectionVolumeReadOnlyCount[v.Collection][stats.NoWriteCanDelete] += 1 } if v.location.isDiskSpaceLow { - collectionVolumeReadOnlyCount[v.Collection]["isDiskSpaceLow"] += 1 + collectionVolumeReadOnlyCount[v.Collection][stats.IsDiskSpaceLow] += 1 } } } @@ -459,6 +459,7 @@ func (s *Store) UnmountVolume(i needle.VolumeId) error { err := location.UnloadVolume(i) if err == nil { glog.V(0).Infof("UnmountVolume %d", i) + stats.DeleteCollectionMetrics(v.Collection) s.DeletedVolumesChan <- message return nil } else if err == ErrVolumeNotFound { From ac5ce312783e545c38ada661b238780af37c86da Mon Sep 17 00:00:00 2001 From: Evgeny Kuzhelev Date: Wed, 27 Jul 2022 16:48:56 +0500 Subject: [PATCH 3/3] leave notion to refactor after prometheus upgrade --- weed/stats/metrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 0f4fcfb52..a2fd373af 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -260,6 +260,7 @@ func SourceName(port uint32) string { return net.JoinHostPort(hostname, strconv.Itoa(int(port))) } +// todo - can be changed to DeletePartialMatch when https://github.com/prometheus/client_golang/pull/1013 gets released func DeleteCollectionMetrics(collection string) { VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal") for _, volume_type := range readOnlyVolumeTypes {