From 9956d93a4023874505eb6e531f930199fa7f3820 Mon Sep 17 00:00:00 2001 From: Dmitry Mishin Date: Mon, 21 Aug 2023 11:37:27 -0700 Subject: [PATCH 01/47] Add disk type to prometheus metrics (#4736) * Add disk type to prometheus metrics * Del metrics * Disk type as readable string --------- Co-authored-by: Dima Mishin --- weed/stats/metrics.go | 8 +++----- weed/storage/erasure_coding/ec_shard.go | 6 ++++-- weed/storage/volume.go | 2 +- weed/storage/volume_loading.go | 2 +- weed/storage/volume_vacuum.go | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index dda4d95e5..af4f26e4c 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -176,7 +176,7 @@ var ( Subsystem: "volumeServer", Name: "volumes", Help: "Number of volumes or shards.", - }, []string{"collection", "type"}) + }, []string{"collection", "type", "disk"}) VolumeServerReadOnlyVolumeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -299,7 +299,6 @@ func JoinHostPort(host string, port int) string { return net.JoinHostPort(host, portStr) } - func StartMetricsServer(ip string, port int) { if port == 0 { return @@ -316,11 +315,10 @@ func SourceName(port uint32) string { return net.JoinHostPort(hostname, strconv.Itoa(int(port))) } -// todo - can be changed to DeletePartialMatch when https://github.com/prometheus/client_golang/pull/1013 gets released func DeleteCollectionMetrics(collection string) { VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal") for _, volume_type := range readOnlyVolumeTypes { - VolumeServerReadOnlyVolumeGauge.DeleteLabelValues(collection, volume_type) + VolumeServerReadOnlyVolumeGauge.DeletePartialMatch(prometheus.Labels{"collection": collection, "type": volume_type}) } - VolumeServerVolumeCounter.DeleteLabelValues(collection, "volume") + VolumeServerVolumeCounter.DeletePartialMatch(prometheus.Labels{"collection": collection, "type": "volume"}) } diff --git a/weed/storage/erasure_coding/ec_shard.go b/weed/storage/erasure_coding/ec_shard.go index 19ee17636..85694e63b 100644 --- a/weed/storage/erasure_coding/ec_shard.go +++ b/weed/storage/erasure_coding/ec_shard.go @@ -7,6 +7,8 @@ import ( "strconv" "strings" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/types" @@ -44,7 +46,7 @@ func NewEcVolumeShard(diskType types.DiskType, dirname string, collection string } v.ecdFileSize = ecdFi.Size() - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "ec_shards").Inc() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "ec_shards", string(v.DiskType)).Inc() return } @@ -88,7 +90,7 @@ func (shard *EcVolumeShard) Close() { func (shard *EcVolumeShard) Destroy() { os.Remove(shard.FileName() + ToExt(int(shard.ShardId))) - stats.VolumeServerVolumeCounter.WithLabelValues(shard.Collection, "ec_shards").Dec() + stats.VolumeServerVolumeCounter.WithLabelValues(shard.Collection, "ec_shards", string(shard.DiskType)).Dec() } func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) { diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 232930b80..8530bc2e7 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -246,7 +246,7 @@ func (v *Volume) doClose() { glog.Warningf("Volume Close fail to sync volume %d", v.Id) } v.DataBackend = nil - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Dec() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume", v.DiskType().ReadableString()).Dec() } } diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index 78dfa6901..e7018d52c 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -201,7 +201,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind } } - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Inc() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume", v.DiskType().ReadableString()).Inc() if err == nil { hasLoadedVolume = true diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 0eaca5ff4..9e5993a82 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -124,7 +124,7 @@ func (v *Volume) CommitCompact() error { } } v.DataBackend = nil - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Dec() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume", v.DiskType().ReadableString()).Dec() var e error if e = v.makeupDiff(v.FileName(".cpd"), v.FileName(".cpx"), v.FileName(".dat"), v.FileName(".idx")); e != nil { From 9215ba24be0aec0c5804927cc73613560e57cbc0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 21 Aug 2023 11:38:12 -0700 Subject: [PATCH 02/47] Revert "Add disk type to prometheus metrics" (#4777) Revert "Add disk type to prometheus metrics (#4736)" This reverts commit 9956d93a4023874505eb6e531f930199fa7f3820. --- weed/stats/metrics.go | 8 +++++--- weed/storage/erasure_coding/ec_shard.go | 6 ++---- weed/storage/volume.go | 2 +- weed/storage/volume_loading.go | 2 +- weed/storage/volume_vacuum.go | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index af4f26e4c..dda4d95e5 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -176,7 +176,7 @@ var ( Subsystem: "volumeServer", Name: "volumes", Help: "Number of volumes or shards.", - }, []string{"collection", "type", "disk"}) + }, []string{"collection", "type"}) VolumeServerReadOnlyVolumeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -299,6 +299,7 @@ func JoinHostPort(host string, port int) string { return net.JoinHostPort(host, portStr) } + func StartMetricsServer(ip string, port int) { if port == 0 { return @@ -315,10 +316,11 @@ func SourceName(port uint32) string { return net.JoinHostPort(hostname, strconv.Itoa(int(port))) } +// todo - can be changed to DeletePartialMatch when https://github.com/prometheus/client_golang/pull/1013 gets released func DeleteCollectionMetrics(collection string) { VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal") for _, volume_type := range readOnlyVolumeTypes { - VolumeServerReadOnlyVolumeGauge.DeletePartialMatch(prometheus.Labels{"collection": collection, "type": volume_type}) + VolumeServerReadOnlyVolumeGauge.DeleteLabelValues(collection, volume_type) } - VolumeServerVolumeCounter.DeletePartialMatch(prometheus.Labels{"collection": collection, "type": "volume"}) + VolumeServerVolumeCounter.DeleteLabelValues(collection, "volume") } diff --git a/weed/storage/erasure_coding/ec_shard.go b/weed/storage/erasure_coding/ec_shard.go index 85694e63b..19ee17636 100644 --- a/weed/storage/erasure_coding/ec_shard.go +++ b/weed/storage/erasure_coding/ec_shard.go @@ -7,8 +7,6 @@ import ( "strconv" "strings" - "github.com/seaweedfs/seaweedfs/weed/storage/types" - "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/types" @@ -46,7 +44,7 @@ func NewEcVolumeShard(diskType types.DiskType, dirname string, collection string } v.ecdFileSize = ecdFi.Size() - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "ec_shards", string(v.DiskType)).Inc() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "ec_shards").Inc() return } @@ -90,7 +88,7 @@ func (shard *EcVolumeShard) Close() { func (shard *EcVolumeShard) Destroy() { os.Remove(shard.FileName() + ToExt(int(shard.ShardId))) - stats.VolumeServerVolumeCounter.WithLabelValues(shard.Collection, "ec_shards", string(shard.DiskType)).Dec() + stats.VolumeServerVolumeCounter.WithLabelValues(shard.Collection, "ec_shards").Dec() } func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) { diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 8530bc2e7..232930b80 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -246,7 +246,7 @@ func (v *Volume) doClose() { glog.Warningf("Volume Close fail to sync volume %d", v.Id) } v.DataBackend = nil - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume", v.DiskType().ReadableString()).Dec() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Dec() } } diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index e7018d52c..78dfa6901 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -201,7 +201,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind } } - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume", v.DiskType().ReadableString()).Inc() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Inc() if err == nil { hasLoadedVolume = true diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 9e5993a82..0eaca5ff4 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -124,7 +124,7 @@ func (v *Volume) CommitCompact() error { } } v.DataBackend = nil - stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume", v.DiskType().ReadableString()).Dec() + stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Dec() var e error if e = v.makeupDiff(v.FileName(".cpd"), v.FileName(".cpx"), v.FileName(".dat"), v.FileName(".idx")); e != nil { From c49caff5a1c98bfcfbf89ea554bd21cd15d71f90 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 21 Aug 2023 23:42:39 -0700 Subject: [PATCH 03/47] change from deprecated jwt.StandardClaims to new jwt.RegisteredClaims --- weed/security/jwt.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/weed/security/jwt.go b/weed/security/jwt.go index 5d9534c7b..0b51f086d 100644 --- a/weed/security/jwt.go +++ b/weed/security/jwt.go @@ -17,14 +17,14 @@ type SigningKey []byte // restricting the access this JWT allows to only a single file. type SeaweedFileIdClaims struct { Fid string `json:"fid"` - jwt.StandardClaims + jwt.RegisteredClaims } // SeaweedFilerClaims is created e.g. by S3 proxy server and consumed by Filer server. // Right now, it only contains the standard claims; but this might be extended later // for more fine-grained permissions. type SeaweedFilerClaims struct { - jwt.StandardClaims + jwt.RegisteredClaims } func GenJwtForVolumeServer(signingKey SigningKey, expiresAfterSec int, fileId string) EncodedJwt { @@ -34,10 +34,10 @@ func GenJwtForVolumeServer(signingKey SigningKey, expiresAfterSec int, fileId st claims := SeaweedFileIdClaims{ fileId, - jwt.StandardClaims{}, + jwt.RegisteredClaims{}, } if expiresAfterSec > 0 { - claims.ExpiresAt = time.Now().Add(time.Second * time.Duration(expiresAfterSec)).Unix() + claims.ExpiresAt = jwt.NewNumericDate(time.Now().Add(time.Second * time.Duration(expiresAfterSec))) } t := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) encoded, e := t.SignedString([]byte(signingKey)) @@ -56,10 +56,10 @@ func GenJwtForFilerServer(signingKey SigningKey, expiresAfterSec int) EncodedJwt } claims := SeaweedFilerClaims{ - jwt.StandardClaims{}, + jwt.RegisteredClaims{}, } if expiresAfterSec > 0 { - claims.ExpiresAt = time.Now().Add(time.Second * time.Duration(expiresAfterSec)).Unix() + claims.ExpiresAt = jwt.NewNumericDate(time.Now().Add(time.Second * time.Duration(expiresAfterSec))) } t := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) encoded, e := t.SignedString([]byte(signingKey)) From a1149f060bbd2d37a92d2d9dec3208a22fcdac7c Mon Sep 17 00:00:00 2001 From: chrislu Date: Tue, 22 Aug 2023 08:26:31 -0700 Subject: [PATCH 04/47] upgrade jwt --- go.mod | 1 + go.sum | 2 ++ weed/security/jwt.go | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 52a61a9c7..eb4ef3c5b 100644 --- a/go.mod +++ b/go.mod @@ -212,6 +212,7 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect diff --git a/go.sum b/go.sum index 84925174e..0fdaef931 100644 --- a/go.sum +++ b/go.sum @@ -331,6 +331,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/weed/security/jwt.go b/weed/security/jwt.go index 0b51f086d..446c3c21d 100644 --- a/weed/security/jwt.go +++ b/weed/security/jwt.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/golang-jwt/jwt" + jwt "github.com/golang-jwt/jwt/v5" "github.com/seaweedfs/seaweedfs/weed/glog" ) From 010c5e91e3287c36451b3fc60550f0f53a15066a Mon Sep 17 00:00:00 2001 From: chrislu Date: Tue, 22 Aug 2023 09:53:54 -0700 Subject: [PATCH 05/47] add stream assign proto --- weed/pb/master.proto | 2 + weed/pb/master_pb/master.pb.go | 283 ++++++++++++++-------------- weed/pb/master_pb/master_grpc.pb.go | 68 +++++++ 3 files changed, 215 insertions(+), 138 deletions(-) diff --git a/weed/pb/master.proto b/weed/pb/master.proto index be4f4a78b..94277104f 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -15,6 +15,8 @@ service Seaweed { } rpc Assign (AssignRequest) returns (AssignResponse) { } + rpc StreamAssign (stream AssignRequest) returns (stream AssignResponse) { + } rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { } rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) { diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 5c0b5d774..4e61faafe 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -4591,7 +4591,7 @@ var file_master_proto_rawDesc = []byte{ 0x72, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x75, 0x66, 0x66, 0x72, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x32, 0xbd, 0x0e, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12, 0x49, 0x0a, 0x0d, + 0x32, 0x88, 0x0f, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12, 0x49, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x14, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, @@ -4611,106 +4611,111 @@ var file_master_proto_rawDesc = []byte{ 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, - 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, + 0x73, 0x73, 0x69, 0x67, 0x6e, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, + 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, + 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, + 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, + 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, + 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, + 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, + 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, + 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, + 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, + 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x43, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, - 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, - 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, - 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, - 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, - 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, - 0x63, 0x75, 0x75, 0x6d, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x45, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, - 0x75, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, - 0x75, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, + 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, + 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, + 0x0a, 0x0d, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x12, + 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, 0x61, + 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x69, 0x73, + 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, + 0x63, 0x75, 0x75, 0x6d, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x24, 0x2e, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, - 0x6c, 0x79, 0x12, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, - 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, - 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, - 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, - 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, + 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, + 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, + 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, + 0x10, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, + 0x73, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, + 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, + 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, + 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, + 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x2e, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x39, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, - 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, - 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, - 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, - 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, - 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, - 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, - 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, - 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, - 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50, 0x69, + 0x6e, 0x67, 0x12, 0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, + 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, + 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, + 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, + 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, + 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, + 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, + 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4837,46 +4842,48 @@ var file_master_proto_depIdxs = []int32{ 8, // 37: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest 12, // 38: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest 15, // 39: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest - 17, // 40: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest - 20, // 41: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest - 22, // 42: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest - 29, // 43: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest - 31, // 44: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest - 33, // 45: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest - 35, // 46: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest - 37, // 47: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest - 39, // 48: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest - 41, // 49: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest - 43, // 50: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest - 45, // 51: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest - 47, // 52: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest - 49, // 53: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest - 55, // 54: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest - 51, // 55: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest - 53, // 56: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest - 1, // 57: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse - 11, // 58: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse - 13, // 59: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse - 16, // 60: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse - 18, // 61: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse - 21, // 62: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse - 23, // 63: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse - 30, // 64: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse - 32, // 65: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse - 34, // 66: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse - 36, // 67: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse - 38, // 68: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse - 40, // 69: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse - 42, // 70: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse - 44, // 71: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse - 46, // 72: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse - 48, // 73: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse - 50, // 74: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse - 56, // 75: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse - 52, // 76: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse - 54, // 77: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse - 57, // [57:78] is the sub-list for method output_type - 36, // [36:57] is the sub-list for method input_type + 15, // 40: master_pb.Seaweed.StreamAssign:input_type -> master_pb.AssignRequest + 17, // 41: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest + 20, // 42: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest + 22, // 43: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest + 29, // 44: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest + 31, // 45: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest + 33, // 46: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest + 35, // 47: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest + 37, // 48: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest + 39, // 49: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest + 41, // 50: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest + 43, // 51: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest + 45, // 52: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest + 47, // 53: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest + 49, // 54: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest + 55, // 55: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest + 51, // 56: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest + 53, // 57: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest + 1, // 58: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse + 11, // 59: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse + 13, // 60: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse + 16, // 61: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse + 16, // 62: master_pb.Seaweed.StreamAssign:output_type -> master_pb.AssignResponse + 18, // 63: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse + 21, // 64: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse + 23, // 65: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse + 30, // 66: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse + 32, // 67: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse + 34, // 68: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse + 36, // 69: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse + 38, // 70: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse + 40, // 71: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse + 42, // 72: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse + 44, // 73: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse + 46, // 74: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse + 48, // 75: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse + 50, // 76: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse + 56, // 77: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse + 52, // 78: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse + 54, // 79: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse + 58, // [58:80] is the sub-list for method output_type + 36, // [36:58] is the sub-list for method input_type 36, // [36:36] is the sub-list for extension type_name 36, // [36:36] is the sub-list for extension extendee 0, // [0:36] is the sub-list for field type_name diff --git a/weed/pb/master_pb/master_grpc.pb.go b/weed/pb/master_pb/master_grpc.pb.go index b0c3f932c..2afcdbbd9 100644 --- a/weed/pb/master_pb/master_grpc.pb.go +++ b/weed/pb/master_pb/master_grpc.pb.go @@ -26,6 +26,7 @@ type SeaweedClient interface { KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) Assign(ctx context.Context, in *AssignRequest, opts ...grpc.CallOption) (*AssignResponse, error) + StreamAssign(ctx context.Context, opts ...grpc.CallOption) (Seaweed_StreamAssignClient, error) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error) CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error) @@ -133,6 +134,37 @@ func (c *seaweedClient) Assign(ctx context.Context, in *AssignRequest, opts ...g return out, nil } +func (c *seaweedClient) StreamAssign(ctx context.Context, opts ...grpc.CallOption) (Seaweed_StreamAssignClient, error) { + stream, err := c.cc.NewStream(ctx, &Seaweed_ServiceDesc.Streams[2], "/master_pb.Seaweed/StreamAssign", opts...) + if err != nil { + return nil, err + } + x := &seaweedStreamAssignClient{stream} + return x, nil +} + +type Seaweed_StreamAssignClient interface { + Send(*AssignRequest) error + Recv() (*AssignResponse, error) + grpc.ClientStream +} + +type seaweedStreamAssignClient struct { + grpc.ClientStream +} + +func (x *seaweedStreamAssignClient) Send(m *AssignRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedStreamAssignClient) Recv() (*AssignResponse, error) { + m := new(AssignResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func (c *seaweedClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) { out := new(StatisticsResponse) err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Statistics", in, out, opts...) @@ -294,6 +326,7 @@ type SeaweedServer interface { KeepConnected(Seaweed_KeepConnectedServer) error LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) Assign(context.Context, *AssignRequest) (*AssignResponse, error) + StreamAssign(Seaweed_StreamAssignServer) error Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error) @@ -330,6 +363,9 @@ func (UnimplementedSeaweedServer) LookupVolume(context.Context, *LookupVolumeReq func (UnimplementedSeaweedServer) Assign(context.Context, *AssignRequest) (*AssignResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Assign not implemented") } +func (UnimplementedSeaweedServer) StreamAssign(Seaweed_StreamAssignServer) error { + return status.Errorf(codes.Unimplemented, "method StreamAssign not implemented") +} func (UnimplementedSeaweedServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented") } @@ -482,6 +518,32 @@ func _Seaweed_Assign_Handler(srv interface{}, ctx context.Context, dec func(inte return interceptor(ctx, in, info, handler) } +func _Seaweed_StreamAssign_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedServer).StreamAssign(&seaweedStreamAssignServer{stream}) +} + +type Seaweed_StreamAssignServer interface { + Send(*AssignResponse) error + Recv() (*AssignRequest, error) + grpc.ServerStream +} + +type seaweedStreamAssignServer struct { + grpc.ServerStream +} + +func (x *seaweedStreamAssignServer) Send(m *AssignResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedStreamAssignServer) Recv() (*AssignRequest, error) { + m := new(AssignRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func _Seaweed_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(StatisticsRequest) if err := dec(in); err != nil { @@ -885,6 +947,12 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{ ServerStreams: true, ClientStreams: true, }, + { + StreamName: "StreamAssign", + Handler: _Seaweed_StreamAssign_Handler, + ServerStreams: true, + ClientStreams: true, + }, }, Metadata: "master.proto", } From ccedad51964a274565d39dafce015c205fd3fb5d Mon Sep 17 00:00:00 2001 From: chrislu Date: Tue, 22 Aug 2023 09:54:06 -0700 Subject: [PATCH 06/47] refactor files --- weed/server/master_grpc_server_assign.go | 103 +++++++++++++++++++++++ weed/server/master_grpc_server_volume.go | 88 ------------------- 2 files changed, 103 insertions(+), 88 deletions(-) create mode 100644 weed/server/master_grpc_server_assign.go diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go new file mode 100644 index 000000000..b55d6a80e --- /dev/null +++ b/weed/server/master_grpc_server_assign.go @@ -0,0 +1,103 @@ +package weed_server + +import ( + "context" + "fmt" + "time" + + "github.com/seaweedfs/raft" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/topology" +) + +func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) { + + if !ms.Topo.IsLeader() { + return nil, raft.NotLeaderError + } + + if req.Count == 0 { + req.Count = 1 + } + + if req.Replication == "" { + req.Replication = ms.option.DefaultReplicaPlacement + } + replicaPlacement, err := super_block.NewReplicaPlacementFromString(req.Replication) + if err != nil { + return nil, err + } + ttl, err := needle.ReadTTL(req.Ttl) + if err != nil { + return nil, err + } + diskType := types.ToDiskType(req.DiskType) + + option := &topology.VolumeGrowOption{ + Collection: req.Collection, + ReplicaPlacement: replicaPlacement, + Ttl: ttl, + DiskType: diskType, + Preallocate: ms.preallocateSize, + DataCenter: req.DataCenter, + Rack: req.Rack, + DataNode: req.DataNode, + MemoryMapMaxSizeMb: req.MemoryMapMaxSizeMb, + } + + vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + + if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(option) { + if ms.Topo.AvailableSpaceFor(option) <= 0 { + return nil, fmt.Errorf("no free volumes left for " + option.String()) + } + vl.AddGrowRequest() + ms.vgCh <- &topology.VolumeGrowRequest{ + Option: option, + Count: int(req.WritableVolumeCount), + } + } + + var ( + lastErr error + maxTimeout = time.Second * 10 + startTime = time.Now() + ) + + for time.Now().Sub(startTime) < maxTimeout { + fid, count, dnList, err := ms.Topo.PickForWrite(req.Count, option) + if err == nil { + dn := dnList.Head() + var replicas []*master_pb.Location + for _, r := range dnList.Rest() { + replicas = append(replicas, &master_pb.Location{ + Url: r.Url(), + PublicUrl: r.PublicUrl, + GrpcPort: uint32(r.GrpcPort), + DataCenter: r.GetDataCenterId(), + }) + } + return &master_pb.AssignResponse{ + Fid: fid, + Location: &master_pb.Location{ + Url: dn.Url(), + PublicUrl: dn.PublicUrl, + GrpcPort: uint32(dn.GrpcPort), + DataCenter: dn.GetDataCenterId(), + }, + Count: count, + Auth: string(security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fid)), + Replicas: replicas, + }, nil + } + //glog.V(4).Infoln("waiting for volume growing...") + lastErr = err + time.Sleep(200 * time.Millisecond) + } + return nil, lastErr +} diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 87c7b9990..4fa6406a7 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -16,7 +16,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" - "github.com/seaweedfs/seaweedfs/weed/topology" ) func (ms *MasterServer) ProcessGrowRequest() { @@ -113,93 +112,6 @@ func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupV return resp, nil } -func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) { - - if !ms.Topo.IsLeader() { - return nil, raft.NotLeaderError - } - - if req.Count == 0 { - req.Count = 1 - } - - if req.Replication == "" { - req.Replication = ms.option.DefaultReplicaPlacement - } - replicaPlacement, err := super_block.NewReplicaPlacementFromString(req.Replication) - if err != nil { - return nil, err - } - ttl, err := needle.ReadTTL(req.Ttl) - if err != nil { - return nil, err - } - diskType := types.ToDiskType(req.DiskType) - - option := &topology.VolumeGrowOption{ - Collection: req.Collection, - ReplicaPlacement: replicaPlacement, - Ttl: ttl, - DiskType: diskType, - Preallocate: ms.preallocateSize, - DataCenter: req.DataCenter, - Rack: req.Rack, - DataNode: req.DataNode, - MemoryMapMaxSizeMb: req.MemoryMapMaxSizeMb, - } - - vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) - - if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(option) { - if ms.Topo.AvailableSpaceFor(option) <= 0 { - return nil, fmt.Errorf("no free volumes left for " + option.String()) - } - vl.AddGrowRequest() - ms.vgCh <- &topology.VolumeGrowRequest{ - Option: option, - Count: int(req.WritableVolumeCount), - } - } - - var ( - lastErr error - maxTimeout = time.Second * 10 - startTime = time.Now() - ) - - for time.Now().Sub(startTime) < maxTimeout { - fid, count, dnList, err := ms.Topo.PickForWrite(req.Count, option) - if err == nil { - dn := dnList.Head() - var replicas []*master_pb.Location - for _, r := range dnList.Rest() { - replicas = append(replicas, &master_pb.Location{ - Url: r.Url(), - PublicUrl: r.PublicUrl, - GrpcPort: uint32(r.GrpcPort), - DataCenter: r.GetDataCenterId(), - }) - } - return &master_pb.AssignResponse{ - Fid: fid, - Location: &master_pb.Location{ - Url: dn.Url(), - PublicUrl: dn.PublicUrl, - GrpcPort: uint32(dn.GrpcPort), - DataCenter: dn.GetDataCenterId(), - }, - Count: count, - Auth: string(security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fid)), - Replicas: replicas, - }, nil - } - //glog.V(4).Infoln("waiting for volume growing...") - lastErr = err - time.Sleep(200 * time.Millisecond) - } - return nil, lastErr -} - func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.StatisticsRequest) (*master_pb.StatisticsResponse, error) { if !ms.Topo.IsLeader() { From 94b7e2a37cc825119121ec41c1da4577470e4040 Mon Sep 17 00:00:00 2001 From: chrislu Date: Tue, 22 Aug 2023 09:59:04 -0700 Subject: [PATCH 07/47] add stream assign server side implementation --- weed/server/master_grpc_server_assign.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index b55d6a80e..34e85d752 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -3,6 +3,7 @@ package weed_server import ( "context" "fmt" + "github.com/seaweedfs/seaweedfs/weed/glog" "time" "github.com/seaweedfs/raft" @@ -15,6 +16,24 @@ import ( "github.com/seaweedfs/seaweedfs/weed/topology" ) +func (ms *MasterServer) StreamAssign(server master_pb.Seaweed_StreamAssignServer) error { + for { + req, err := server.Recv() + if err != nil { + glog.Errorf("StreamAssign failed to receive: %v", err) + return err + } + resp, err := ms.Assign(context.Background(), req) + if err != nil { + glog.Errorf("StreamAssign failed to assign: %v", err) + return err + } + if err = server.Send(resp); err != nil { + glog.Errorf("StreamAssign failed to send: %v", err) + return err + } + } +} func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) { if !ms.Topo.IsLeader() { From d7937f973812c7ae88c4a2a164ca71bc55802dc8 Mon Sep 17 00:00:00 2001 From: gfhe <90042648+gfhe@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:11:11 +0800 Subject: [PATCH 08/47] fix bugs: seaweedfs master ingress host configuration do not work (#4781) fix bugs: seaweedfs hem charts ignore the master ingress configuration --- k8s/charts/seaweedfs/templates/ingress.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/charts/seaweedfs/templates/ingress.yaml b/k8s/charts/seaweedfs/templates/ingress.yaml index d14adf1f3..ab46253fd 100644 --- a/k8s/charts/seaweedfs/templates/ingress.yaml +++ b/k8s/charts/seaweedfs/templates/ingress.yaml @@ -52,7 +52,8 @@ metadata: spec: ingressClassName: {{ .Values.master.ingress.className | quote }} rules: - - http: + - host: {{ .Values.master.ingress.host }} + http: paths: - path: /sw-master/?(.*) pathType: ImplementationSpecific From 99f037b958b5952628d281461d3bfb76fa433d8c Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 23 Aug 2023 00:31:33 -0700 Subject: [PATCH 09/47] streaming assign file ids --- weed/operation/assign_file_id.go | 105 +++++++++++++++++++++++++- weed/operation/assign_file_id_test.go | 68 +++++++++++++++++ weed/server/filer_server.go | 7 +- 3 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 weed/operation/assign_file_id_test.go diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index c2f5a806d..a4753d234 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -4,11 +4,10 @@ import ( "context" "fmt" "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/storage/needle" - "google.golang.org/grpc" - "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "google.golang.org/grpc" ) type VolumeAssignRequest struct { @@ -34,6 +33,106 @@ type AssignResult struct { Replicas []Location `json:"replicas,omitempty"` } +// This is a proxy to the master server, only for assigning volume ids. +// It runs via grpc to the master server in streaming mode. +// The connection to the master would only be re-established when the last connection has error. +type AssignProxy struct { + grpcConnection *grpc.ClientConn + pool chan *singleThreadAssignProxy +} + +func NewAssignProxy(masterFn GetMasterFn, grpcDialOption grpc.DialOption, concurrency int) (ap *AssignProxy, err error) { + ap = &AssignProxy{ + pool: make(chan *singleThreadAssignProxy, concurrency), + } + ap.grpcConnection, err = pb.GrpcDial(context.Background(), masterFn().ToGrpcAddress(), true, grpcDialOption) + if err != nil { + return nil, fmt.Errorf("fail to dial %s: %v", masterFn().ToGrpcAddress(), err) + } + for i := 0; i < concurrency; i++ { + ap.pool <- &singleThreadAssignProxy{} + } + return ap, nil +} + +func (ap *AssignProxy) Assign(primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (ret *AssignResult, err error) { + p := <-ap.pool + defer func() { + ap.pool <- p + }() + + return p.doAssign(ap.grpcConnection, primaryRequest, alternativeRequests...) +} + +type singleThreadAssignProxy struct { + assignClient master_pb.Seaweed_StreamAssignClient +} + +func (ap *singleThreadAssignProxy) doAssign(grpcConnection *grpc.ClientConn, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (ret *AssignResult, err error) { + if ap.assignClient == nil { + client := master_pb.NewSeaweedClient(grpcConnection) + ap.assignClient, err = client.StreamAssign(context.Background()) + if err != nil { + ap.assignClient = nil + return nil, fmt.Errorf("fail to create stream assign client: %v", err) + } + } + + var requests []*VolumeAssignRequest + requests = append(requests, primaryRequest) + requests = append(requests, alternativeRequests...) + ret = &AssignResult{} + + for _, request := range requests { + if request == nil { + continue + } + req := &master_pb.AssignRequest{ + Count: request.Count, + Replication: request.Replication, + Collection: request.Collection, + Ttl: request.Ttl, + DiskType: request.DiskType, + DataCenter: request.DataCenter, + Rack: request.Rack, + DataNode: request.DataNode, + WritableVolumeCount: request.WritableVolumeCount, + } + if err = ap.assignClient.Send(req); err != nil { + return nil, fmt.Errorf("StreamAssignSend: %v", err) + } + resp, grpcErr := ap.assignClient.Recv() + if grpcErr != nil { + return nil, grpcErr + } + if resp.Error != "" { + return nil, fmt.Errorf("StreamAssignRecv: %v", resp.Error) + } + + ret.Count = resp.Count + ret.Fid = resp.Fid + ret.Url = resp.Location.Url + ret.PublicUrl = resp.Location.PublicUrl + ret.GrpcPort = int(resp.Location.GrpcPort) + ret.Error = resp.Error + ret.Auth = security.EncodedJwt(resp.Auth) + for _, r := range resp.Replicas { + ret.Replicas = append(ret.Replicas, Location{ + Url: r.Url, + PublicUrl: r.PublicUrl, + DataCenter: r.DataCenter, + }) + } + + if ret.Count <= 0 { + continue + } + break + } + + return +} + func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) { var requests []*VolumeAssignRequest diff --git a/weed/operation/assign_file_id_test.go b/weed/operation/assign_file_id_test.go new file mode 100644 index 000000000..f6362dceb --- /dev/null +++ b/weed/operation/assign_file_id_test.go @@ -0,0 +1,68 @@ +package operation + +import ( + "fmt" + "github.com/seaweedfs/seaweedfs/weed/pb" + "google.golang.org/grpc" + "testing" + "time" +) + +func BenchmarkWithConcurrency(b *testing.B) { + concurrencyLevels := []int{1, 10, 100, 1000} + + ap, _ := NewAssignProxy(func() pb.ServerAddress { + return pb.ServerAddress("localhost:9333") + }, grpc.WithInsecure(), 16) + + for _, concurrency := range concurrencyLevels { + b.Run( + fmt.Sprintf("Concurrency-%d", concurrency), + func(b *testing.B) { + for i := 0; i < b.N; i++ { + done := make(chan struct{}) + startTime := time.Now() + + for j := 0; j < concurrency; j++ { + go func() { + + ap.Assign(&VolumeAssignRequest{ + Count: 1, + }) + + done <- struct{}{} + }() + } + + for j := 0; j < concurrency; j++ { + <-done + } + + duration := time.Since(startTime) + b.Logf("Concurrency: %d, Duration: %v", concurrency, duration) + } + }, + ) + } +} + +func BenchmarkStreamAssign(b *testing.B) { + ap, _ := NewAssignProxy(func() pb.ServerAddress { + return pb.ServerAddress("localhost:9333") + }, grpc.WithInsecure(), 16) + for i := 0; i < b.N; i++ { + ap.Assign(&VolumeAssignRequest{ + Count: 1, + }) + } +} + +func BenchmarkUnaryAssign(b *testing.B) { + for i := 0; i < b.N; i++ { + Assign(func() pb.ServerAddress { + return pb.ServerAddress("localhost:9333") + }, grpc.WithInsecure(), &VolumeAssignRequest{ + Count: 1, + }) + } +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 8a6d341bb..8e40b2145 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -94,6 +94,9 @@ type FilerServer struct { // track known metadata listeners knownListenersLock sync.Mutex knownListeners map[int32]int32 + + // client to assign file id + assignProxy *operation.AssignProxy } func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) { @@ -131,6 +134,8 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) go stats.LoopPushingMetric("filer", string(fs.option.Host), fs.metricsAddress, fs.metricsIntervalSec) go fs.filer.KeepMasterClientConnected() + fs.assignProxy, err = operation.NewAssignProxy(fs.filer.GetMaster, fs.grpcDialOption, 16) + if !util.LoadConfiguration("filer", false) { v.SetDefault("leveldb2.enabled", true) v.SetDefault("leveldb2.dir", option.DefaultLevelDbDir) @@ -183,7 +188,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs.filer.Dlm.LockRing.SetTakeSnapshotCallback(fs.OnDlmChangeSnapshot) - return fs, nil + return fs, err } func (fs *FilerServer) checkWithMaster() { From 733db2bc8846b98bb191e0cacc1e910104a0cfde Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 23 Aug 2023 00:34:21 -0700 Subject: [PATCH 10/47] turn on streaming assign file id --- weed/server/filer_server_handlers_write.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 898975d14..daf63fa8d 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -43,7 +43,7 @@ func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, u ar, altRequest := so.ToAssignRequests(1) - assignResult, ae := operation.Assign(fs.filer.GetMaster, fs.grpcDialOption, ar, altRequest) + assignResult, ae := fs.assignProxy.Assign(ar, altRequest) if ae != nil { glog.Errorf("failing to assign a file id: %v", ae) err = ae From 5251b4d50ea55c5f0a2fbc60206785bf80775bac Mon Sep 17 00:00:00 2001 From: tikhon-opsfleet <112847337+tikhon-opsfleet@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:01:41 +0600 Subject: [PATCH 11/47] [helm] add possibility to use array for imagePullSecrets (#4782) add possibility to use array for imagePullSecrets --- k8s/charts/seaweedfs/templates/_helpers.tpl | 17 ++++++++++++++++- .../seaweedfs/templates/filer-statefulset.yaml | 5 +---- .../seaweedfs/templates/master-statefulset.yaml | 5 +---- .../seaweedfs/templates/s3-deployment.yaml | 5 +---- .../seaweedfs/templates/volume-statefulset.yaml | 5 +---- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/k8s/charts/seaweedfs/templates/_helpers.tpl b/k8s/charts/seaweedfs/templates/_helpers.tpl index b8048abb6..712657474 100644 --- a/k8s/charts/seaweedfs/templates/_helpers.tpl +++ b/k8s/charts/seaweedfs/templates/_helpers.tpl @@ -182,4 +182,19 @@ Inject extra environment vars in the format key:value, if populated {{- else -}} {{- printf "false" -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{/* Return the proper imagePullSecrets */}} +{{- define "seaweedfs.imagePullSecrets" -}} +{{- if .Values.global.imagePullSecrets }} +{{- if kindIs "string" .Values.global.imagePullSecrets }} +imagePullSecrets: + - name: {{ .Values.global.imagePullSecrets }} +{{- else }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index b770025a1..7a2810a71 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -42,10 +42,7 @@ spec: tolerations: {{ tpl .Values.filer.tolerations . | nindent 8 | trim }} {{- end }} - {{- if .Values.global.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.global.imagePullSecrets }} - {{- end }} + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} {{- if .Values.global.createClusterRole }} serviceAccountName: seaweedfs-rw-sa #hack for delete pod master after migration {{- end }} diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index 7a01eff85..e1b272ab4 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -42,10 +42,7 @@ spec: tolerations: {{ tpl .Values.master.tolerations . | nindent 8 | trim }} {{- end }} - {{- if .Values.global.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.global.imagePullSecrets }} - {{- end }} + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} terminationGracePeriodSeconds: 60 {{- if .Values.master.priorityClassName }} priorityClassName: {{ .Values.master.priorityClassName | quote }} diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index 8565b6c81..d0d56d0ef 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -30,10 +30,7 @@ spec: tolerations: {{ tpl .Values.s3.tolerations . | nindent 8 | trim }} {{- end }} - {{- if .Values.global.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.global.imagePullSecrets }} - {{- end }} + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} terminationGracePeriodSeconds: 10 {{- if .Values.s3.priorityClassName }} priorityClassName: {{ .Values.s3.priorityClassName | quote }} diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 4c4681618..4abadb24d 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -36,10 +36,7 @@ spec: tolerations: {{ tpl .Values.volume.tolerations . | nindent 8 | trim }} {{- end }} - {{- if .Values.global.imagePullSecrets }} - imagePullSecrets: - - name: {{ .Values.global.imagePullSecrets }} - {{- end }} + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} terminationGracePeriodSeconds: 150 {{- if .Values.volume.priorityClassName }} priorityClassName: {{ .Values.volume.priorityClassName | quote }} From 796b7508f33916f7fa734e3df2ceea9a80415ade Mon Sep 17 00:00:00 2001 From: Nico D'Cotta <45274424+Cottand@users.noreply.github.com> Date: Thu, 24 Aug 2023 16:08:56 +0200 Subject: [PATCH 12/47] Implement SRV lookups for filer (#4767) --- weed/command/benchmark.go | 2 +- weed/command/filer.go | 6 +-- weed/command/server.go | 4 +- weed/filer/filer.go | 3 +- weed/filer/leveldb/leveldb_store_test.go | 7 +-- weed/filer/leveldb2/leveldb2_store_test.go | 5 +- weed/filer/leveldb3/leveldb3_store_test.go | 5 +- weed/filer/rocksdb/rocksdb_store_test.go | 6 +-- weed/iamapi/iamapi_server.go | 2 +- weed/mq/broker/broker_server.go | 2 +- weed/pb/server_address.go | 37 +++++++++++++ weed/pb/server_address_test.go | 36 +++++++++++++ weed/pb/server_discovery.go | 62 ++++++++++++++++++++++ weed/server/filer_grpc_server_admin.go | 2 +- weed/server/filer_server.go | 10 ++-- weed/server/master_server.go | 2 +- weed/shell/commands.go | 2 +- weed/stats/metrics.go | 1 - weed/wdclient/masterclient.go | 15 +++--- weed/wdclient/vid_map_test.go | 5 +- 20 files changed, 177 insertions(+), 37 deletions(-) create mode 100644 weed/pb/server_address_test.go create mode 100644 weed/pb/server_discovery.go diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go index 7f132892e..7f9a23cf8 100644 --- a/weed/command/benchmark.go +++ b/weed/command/benchmark.go @@ -127,7 +127,7 @@ func runBenchmark(cmd *Command, args []string) bool { defer pprof.StopCPUProfile() } - b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", pb.ServerAddresses(*b.masters).ToAddressMap()) + b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", *pb.ServerAddresses(*b.masters).ToServiceDiscovery()) go b.masterClient.KeepConnectedToMaster() b.masterClient.WaitUntilConnected() diff --git a/weed/command/filer.go b/weed/command/filer.go index 83e2abdac..7e636974f 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -33,7 +33,7 @@ var ( ) type FilerOptions struct { - masters map[string]pb.ServerAddress + masters *pb.ServerDiscovery mastersString *string ip *string bindIp *string @@ -65,7 +65,7 @@ type FilerOptions struct { func init() { cmdFiler.Run = runFiler // break init cycle - f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers") + f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers or a single DNS SRV record of at least 1 master server, prepended with dnssrv+") f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup") f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection") f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address") @@ -208,7 +208,7 @@ func runFiler(cmd *Command, args []string) bool { }(startDelay) } - f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap() + f.masters = pb.ServerAddresses(*f.mastersString).ToServiceDiscovery() f.startFiler() diff --git a/weed/command/server.go b/weed/command/server.go index fecb1cad6..7fbb59676 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -203,7 +203,7 @@ func runServer(cmd *Command, args []string) bool { // ip address masterOptions.ip = serverIp masterOptions.ipBind = serverBindIp - filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap() + filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToServiceDiscovery() filerOptions.ip = serverIp filerOptions.bindIp = serverBindIp s3Options.bindIp = serverBindIp @@ -216,7 +216,7 @@ func runServer(cmd *Command, args []string) bool { serverOptions.v.dataCenter = serverDataCenter serverOptions.v.rack = serverRack mqBrokerOptions.ip = serverIp - mqBrokerOptions.masters = filerOptions.masters + mqBrokerOptions.masters = filerOptions.masters.GetInstancesAsMap() mqBrokerOptions.filerGroup = filerOptions.filerGroup // serverOptions.v.pulseSeconds = pulseSeconds diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 8570faa7a..fdc425f07 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -52,8 +52,7 @@ type Filer struct { Dlm *lock_manager.DistributedLockManager } -func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, - filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer { +func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer { f := &Filer{ MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters), fileIdDeletionQueue: util.NewUnboundedQueue(), diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index 7013f67a7..c8e71a003 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -3,6 +3,7 @@ package leveldb import ( "context" "fmt" + "github.com/seaweedfs/seaweedfs/weed/pb" "os" "testing" "time" @@ -12,7 +13,7 @@ import ( ) func TestCreateAndFind(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil) dir := t.TempDir() store := &LevelDBStore{} store.initialize(dir) @@ -65,7 +66,7 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil) dir := t.TempDir() store := &LevelDBStore{} store.initialize(dir) @@ -87,7 +88,7 @@ func TestEmptyRoot(t *testing.T) { } func BenchmarkInsertEntry(b *testing.B) { - testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil) dir := b.TempDir() store := &LevelDBStore{} store.initialize(dir) diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index f7ec99e06..b25dcc7b8 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -2,6 +2,7 @@ package leveldb import ( "context" + "github.com/seaweedfs/seaweedfs/weed/pb" "testing" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -9,7 +10,7 @@ import ( ) func TestCreateAndFind(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil) dir := t.TempDir() store := &LevelDB2Store{} store.initialize(dir, 2) @@ -62,7 +63,7 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil) dir := t.TempDir() store := &LevelDB2Store{} store.initialize(dir, 2) diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index e2e4d5099..a2d8dd8a3 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -2,6 +2,7 @@ package leveldb import ( "context" + "github.com/seaweedfs/seaweedfs/weed/pb" "testing" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -9,7 +10,7 @@ import ( ) func TestCreateAndFind(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil) dir := t.TempDir() store := &LevelDB3Store{} store.initialize(dir) @@ -62,7 +63,7 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil) dir := t.TempDir() store := &LevelDB3Store{} store.initialize(dir) diff --git a/weed/filer/rocksdb/rocksdb_store_test.go b/weed/filer/rocksdb/rocksdb_store_test.go index e89327baa..e24274d2a 100644 --- a/weed/filer/rocksdb/rocksdb_store_test.go +++ b/weed/filer/rocksdb/rocksdb_store_test.go @@ -15,7 +15,7 @@ import ( ) func TestCreateAndFind(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", 0, "", "", "", nil) dir := t.TempDir() store := &RocksDBStore{} store.initialize(dir) @@ -68,7 +68,7 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", 0, "", "", "", nil) dir := t.TempDir() store := &RocksDBStore{} store.initialize(dir) @@ -90,7 +90,7 @@ func TestEmptyRoot(t *testing.T) { } func BenchmarkInsertEntry(b *testing.B) { - testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) + testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", 0, "", "", "", nil) dir := b.TempDir() store := &RocksDBStore{} store.initialize(dir) diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index 223bcb296..63d2e7a75 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -50,7 +50,7 @@ var s3ApiConfigure IamS3ApiConfig func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer *IamApiServer, err error) { s3ApiConfigure = IamS3ApiConfigure{ option: option, - masterClient: wdclient.NewMasterClient(option.GrpcDialOption, "", "iam", "", "", "", option.Masters), + masterClient: wdclient.NewMasterClient(option.GrpcDialOption, "", "iam", "", "", "", *pb.NewServiceDiscoveryFromMap(option.Masters)), } s3Option := s3api.S3ApiServerOption{Filer: option.Filer} iamApiServer = &IamApiServer{ diff --git a/weed/mq/broker/broker_server.go b/weed/mq/broker/broker_server.go index 4f5b3c28d..da1284c80 100644 --- a/weed/mq/broker/broker_server.go +++ b/weed/mq/broker/broker_server.go @@ -41,7 +41,7 @@ func NewMessageBroker(option *MessageQueueBrokerOption, grpcDialOption grpc.Dial mqBroker = &MessageQueueBroker{ option: option, grpcDialOption: grpcDialOption, - MasterClient: wdclient.NewMasterClient(grpcDialOption, option.FilerGroup, cluster.BrokerType, pb.NewServerAddress(option.Ip, option.Port, 0), option.DataCenter, option.Rack, option.Masters), + MasterClient: wdclient.NewMasterClient(grpcDialOption, option.FilerGroup, cluster.BrokerType, pb.NewServerAddress(option.Ip, option.Port, 0), option.DataCenter, option.Rack, *pb.NewServiceDiscoveryFromMap(option.Masters)), filers: make(map[pb.ServerAddress]struct{}), localTopicManager: topic.NewLocalTopicManager(), } diff --git a/weed/pb/server_address.go b/weed/pb/server_address.go index 56d0dba24..a0aa79ae4 100644 --- a/weed/pb/server_address.go +++ b/weed/pb/server_address.go @@ -11,6 +11,7 @@ import ( type ServerAddress string type ServerAddresses string +type ServerSrvAddress string func NewServerAddress(host string, port int, grpcPort int) ServerAddress { if grpcPort == 0 || grpcPort == port+10000 { @@ -76,6 +77,42 @@ func (sa ServerAddress) ToGrpcAddress() string { return ServerToGrpcAddress(string(sa)) } +// LookUp may return an error for some records along with successful lookups - make sure you do not +// discard `addresses` even if `err == nil` +func (r ServerSrvAddress) LookUp() (addresses []ServerAddress, err error) { + _, records, lookupErr := net.LookupSRV("", "", string(r)) + if lookupErr != nil { + err = fmt.Errorf("lookup SRV address %s: %v", r, lookupErr) + } + for _, srv := range records { + address := fmt.Sprintf("%s:%d", srv.Target, srv.Port) + addresses = append(addresses, ServerAddress(address)) + } + return +} + +// ToServiceDiscovery expects one of: a comma-separated list of ip:port, like +// +// 10.0.0.1:9999,10.0.0.2:24:9999 +// +// OR an SRV Record prepended with 'dnssrv+', like: +// +// dnssrv+_grpc._tcp.master.consul +// dnssrv+_grpc._tcp.headless.default.svc.cluster.local +// dnssrv+seaweed-master.master.consul +func (sa ServerAddresses) ToServiceDiscovery() (sd *ServerDiscovery) { + sd = &ServerDiscovery{} + prefix := "dnssrv+" + if strings.HasPrefix(string(sa), prefix) { + trimmed := strings.TrimPrefix(string(sa), prefix) + srv := ServerSrvAddress(trimmed) + sd.srvRecord = &srv + } else { + sd.list = sa.ToAddresses() + } + return +} + func (sa ServerAddresses) ToAddresses() (addresses []ServerAddress) { parts := strings.Split(string(sa), ",") for _, address := range parts { diff --git a/weed/pb/server_address_test.go b/weed/pb/server_address_test.go new file mode 100644 index 000000000..f5a12427a --- /dev/null +++ b/weed/pb/server_address_test.go @@ -0,0 +1,36 @@ +package pb + +import ( + "reflect" + "testing" +) + +func TestServerAddresses_ToAddressMapOrSrv_shouldRemovePrefix(t *testing.T) { + str := ServerAddresses("dnssrv+hello.srv.consul") + + d := str.ToServiceDiscovery() + + expected := ServerSrvAddress("hello.srv.consul") + if *d.srvRecord != expected { + t.Fatalf(`ServerAddresses("dnssrv+hello.srv.consul") = %s, expected %s`, *d.srvRecord, expected) + } +} + +func TestServerAddresses_ToAddressMapOrSrv_shouldHandleIPPortList(t *testing.T) { + str := ServerAddresses("10.0.0.1:23,10.0.0.2:24") + + d := str.ToServiceDiscovery() + + if d.srvRecord != nil { + t.Fatalf(`ServerAddresses("dnssrv+hello.srv.consul") = %s, expected nil`, *d.srvRecord) + } + + expected := []ServerAddress{ + ServerAddress("10.0.0.1:23"), + ServerAddress("10.0.0.2:24"), + } + + if !reflect.DeepEqual(d.list, expected) { + t.Fatalf(`Expected %q, got %q`, expected, d.list) + } +} diff --git a/weed/pb/server_discovery.go b/weed/pb/server_discovery.go new file mode 100644 index 000000000..25c0360c5 --- /dev/null +++ b/weed/pb/server_discovery.go @@ -0,0 +1,62 @@ +package pb + +import ( + "github.com/seaweedfs/seaweedfs/weed/glog" + "reflect" +) + +// ServerDiscovery encodes a way to find at least 1 instance of a service, +// and provides utility functions to refresh the instance list +type ServerDiscovery struct { + list []ServerAddress + srvRecord *ServerSrvAddress +} + +func NewServiceDiscoveryFromMap(m map[string]ServerAddress) (sd *ServerDiscovery) { + sd = &ServerDiscovery{} + for _, s := range m { + sd.list = append(sd.list, s) + } + return sd +} + +// RefreshBySrvIfAvailable performs a DNS SRV lookup and updates list with the results +// of the lookup +func (sd *ServerDiscovery) RefreshBySrvIfAvailable() { + if sd.srvRecord == nil { + return + } + newList, err := sd.srvRecord.LookUp() + if err != nil { + glog.V(0).Infof("failed to lookup SRV for %s: %v", *sd.srvRecord, err) + } + if newList == nil || len(newList) == 0 { + glog.V(0).Infof("looked up SRV for %s, but found no well-formed names", *sd.srvRecord) + return + } + if !reflect.DeepEqual(sd.list, newList) { + sd.list = newList + } +} + +// GetInstances returns a copy of the latest known list of addresses +// call RefreshBySrvIfAvailable prior to this in order to get a more up-to-date view +func (sd *ServerDiscovery) GetInstances() (addresses []ServerAddress) { + for _, a := range sd.list { + addresses = append(addresses, a) + } + return addresses +} +func (sd *ServerDiscovery) GetInstancesAsStrings() (addresses []string) { + for _, i := range sd.list { + addresses = append(addresses, string(i)) + } + return addresses +} +func (sd *ServerDiscovery) GetInstancesAsMap() (addresses map[string]ServerAddress) { + addresses = make(map[string]ServerAddress) + for _, i := range sd.list { + addresses[string(i)] = i + } + return addresses +} diff --git a/weed/server/filer_grpc_server_admin.go b/weed/server/filer_grpc_server_admin.go index 58215a927..8a58e287c 100644 --- a/weed/server/filer_grpc_server_admin.go +++ b/weed/server/filer_grpc_server_admin.go @@ -87,7 +87,7 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb. clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId")) t := &filer_pb.GetFilerConfigurationResponse{ - Masters: pb.ToAddressStringsFromMap(fs.option.Masters), + Masters: fs.option.Masters.GetInstancesAsStrings(), Collection: fs.option.Collection, Replication: fs.option.DefaultReplication, MaxMb: uint32(fs.option.MaxMB), diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 8e40b2145..98784bce3 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -50,7 +50,7 @@ import ( ) type FilerOption struct { - Masters map[string]pb.ServerAddress + Masters *pb.ServerDiscovery FilerGroup string Collection string DefaultReplication string @@ -118,11 +118,12 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) } fs.listenersCond = sync.NewCond(&fs.listenersLock) - if len(option.Masters) == 0 { + option.Masters.RefreshBySrvIfAvailable() + if len(option.Masters.GetInstances()) == 0 { glog.Fatal("master list is required!") } - fs.filer = filer.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.FilerGroup, option.Collection, option.DefaultReplication, option.DataCenter, func() { + fs.filer = filer.NewFiler(*option.Masters, fs.grpcDialOption, option.Host, option.FilerGroup, option.Collection, option.DefaultReplication, option.DataCenter, func() { fs.listenersCond.Broadcast() }) fs.filer.Cipher = option.Cipher @@ -195,7 +196,8 @@ func (fs *FilerServer) checkWithMaster() { isConnected := false for !isConnected { - for _, master := range fs.option.Masters { + fs.option.Masters.RefreshBySrvIfAvailable() + for _, master := range fs.option.Masters.GetInstances() { readErr := operation.WithMasterServerClient(false, master, fs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error { resp, err := masterClient.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{}) if err != nil { diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 2489aaefd..9a5313a10 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -110,7 +110,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se vgCh: make(chan *topology.VolumeGrowRequest, 1<<6), clientChans: make(map[string]chan *master_pb.KeepConnectedResponse), grpcDialOption: grpcDialOption, - MasterClient: wdclient.NewMasterClient(grpcDialOption, "", cluster.MasterType, option.Master, "", "", peers), + MasterClient: wdclient.NewMasterClient(grpcDialOption, "", cluster.MasterType, option.Master, "", "", *pb.NewServiceDiscoveryFromMap(peers)), adminLocks: NewAdminLocks(), Cluster: cluster.NewCluster(), } diff --git a/weed/shell/commands.go b/weed/shell/commands.go index b1722edfb..e6e582376 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -51,7 +51,7 @@ var ( func NewCommandEnv(options *ShellOptions) *CommandEnv { ce := &CommandEnv{ env: make(map[string]string), - MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, *options.FilerGroup, pb.AdminShellClient, "", "", "", pb.ServerAddresses(*options.Masters).ToAddressMap()), + MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, *options.FilerGroup, pb.AdminShellClient, "", "", "", *pb.ServerAddresses(*options.Masters).ToServiceDiscovery()), option: options, } ce.locker = exclusive_locks.NewExclusiveLocker(ce.MasterClient, "shell") diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index dda4d95e5..3dda42423 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -299,7 +299,6 @@ func JoinHostPort(host string, port int) string { return net.JoinHostPort(host, portStr) } - func StartMetricsServer(ip string, port int) { if port == 0 { return diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index c693df582..a6ddf22f3 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -24,8 +24,8 @@ type MasterClient struct { rack string currentMaster pb.ServerAddress currentMasterLock sync.RWMutex - masters map[string]pb.ServerAddress - grpcDialOption grpc.DialOption + masters pb.ServerDiscovery + grpcDialOption grpc.DialOption *vidMap vidMapCacheSize int @@ -33,7 +33,7 @@ type MasterClient struct { OnPeerUpdateLock sync.RWMutex } -func NewMasterClient(grpcDialOption grpc.DialOption, filerGroup string, clientType string, clientHost pb.ServerAddress, clientDataCenter string, rack string, masters map[string]pb.ServerAddress) *MasterClient { +func NewMasterClient(grpcDialOption grpc.DialOption, filerGroup string, clientType string, clientHost pb.ServerAddress, clientDataCenter string, rack string, masters pb.ServerDiscovery) *MasterClient { return &MasterClient{ FilerGroup: filerGroup, clientType: clientType, @@ -108,9 +108,9 @@ func (mc *MasterClient) GetMaster() pb.ServerAddress { return mc.getCurrentMaster() } -func (mc *MasterClient) GetMasters() map[string]pb.ServerAddress { +func (mc *MasterClient) GetMasters() []pb.ServerAddress { mc.WaitUntilConnected() - return mc.masters + return mc.masters.GetInstances() } func (mc *MasterClient) WaitUntilConnected() { @@ -132,7 +132,7 @@ func (mc *MasterClient) KeepConnectedToMaster() { } func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddress) (leader string) { - for _, master := range mc.masters { + for _, master := range mc.masters.GetInstances() { if master == myMasterAddress { continue } @@ -159,7 +159,8 @@ func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddres func (mc *MasterClient) tryAllMasters() { var nextHintedLeader pb.ServerAddress - for _, master := range mc.masters { + mc.masters.RefreshBySrvIfAvailable() + for _, master := range mc.masters.GetInstances() { nextHintedLeader = mc.tryConnectToMaster(master) for nextHintedLeader != "" { nextHintedLeader = mc.tryConnectToMaster(nextHintedLeader) diff --git a/weed/wdclient/vid_map_test.go b/weed/wdclient/vid_map_test.go index 980e5bd8c..a734c6b0c 100644 --- a/weed/wdclient/vid_map_test.go +++ b/weed/wdclient/vid_map_test.go @@ -3,6 +3,7 @@ package wdclient import ( "context" "fmt" + "github.com/seaweedfs/seaweedfs/weed/pb" "google.golang.org/grpc" "strconv" "sync" @@ -65,7 +66,7 @@ func TestLocationIndex(t *testing.T) { } func TestLookupFileId(t *testing.T) { - mc := NewMasterClient(grpc.EmptyDialOption{}, "", "", "", "", "", nil) + mc := NewMasterClient(grpc.EmptyDialOption{}, "", "", "", "", "", pb.ServerDiscovery{}) length := 5 //Construct a cache linked list of length 5 @@ -135,7 +136,7 @@ func TestLookupFileId(t *testing.T) { } func TestConcurrentGetLocations(t *testing.T) { - mc := NewMasterClient(grpc.EmptyDialOption{}, "", "", "", "", "", nil) + mc := NewMasterClient(grpc.EmptyDialOption{}, "", "", "", "", "", pb.ServerDiscovery{}) location := Location{Url: "TestDataRacing"} mc.addLocation(1, location) From 3d45ecd326c42533a8fc8b40a9310f55d182adb7 Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Sun, 27 Aug 2023 23:53:28 +0300 Subject: [PATCH 13/47] [helm] fix log to stderr and default host for ingress (#4787) --- k8s/charts/seaweedfs/templates/ingress.yaml | 12 ++++++++---- k8s/charts/seaweedfs/templates/s3-deployment.yaml | 2 ++ k8s/charts/seaweedfs/values.yaml | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/k8s/charts/seaweedfs/templates/ingress.yaml b/k8s/charts/seaweedfs/templates/ingress.yaml index ab46253fd..183cf1a23 100644 --- a/k8s/charts/seaweedfs/templates/ingress.yaml +++ b/k8s/charts/seaweedfs/templates/ingress.yaml @@ -17,8 +17,7 @@ spec: tls: {{ .Values.filer.ingress.tls | default list | toYaml | nindent 6}} rules: - - host: {{ .Values.filer.ingress.host }} - http: + - http: paths: - path: /sw-filer/?(.*) pathType: ImplementationSpecific @@ -33,6 +32,9 @@ spec: serviceName: {{ template "seaweedfs.name" . }}-filer servicePort: {{ .Values.filer.port }} {{- end }} +{{- if .Values.filer.ingress.host }} + host: {{ .Values.filer.ingress.host }} +{{- end }} {{- end }} --- {{- if .Values.master.ingress.enabled }} @@ -52,8 +54,7 @@ metadata: spec: ingressClassName: {{ .Values.master.ingress.className | quote }} rules: - - host: {{ .Values.master.ingress.host }} - http: + - http: paths: - path: /sw-master/?(.*) pathType: ImplementationSpecific @@ -68,4 +69,7 @@ spec: serviceName: {{ template "seaweedfs.name" . }}-master servicePort: {{ .Values.master.port }} {{- end }} +{{- if .Values.filer.ingress.host }} + host: {{ .Values.master.ingress.host }} +{{- end }} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index d0d56d0ef..2040e7161 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -107,8 +107,10 @@ spec: {{- end }} -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }} volumeMounts: + {{- if eq .Values.s3.logs.type "hostPath" }} - name: logs mountPath: "/logs/" + {{- end }} - mountPath: /etc/sw name: config-users readOnly: true diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 8a4148102..165be23f3 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -127,6 +127,7 @@ master: ingress: enabled: false className: "nginx" + # host: false for "*" hostname host: "master.seaweedfs.local" annotations: nginx.ingress.kubernetes.io/auth-type: "basic" @@ -375,6 +376,7 @@ filer: ingress: enabled: false className: "nginx" + # host: false for "*" hostname host: "seaweedfs.cluster.local" annotations: nginx.ingress.kubernetes.io/backend-protocol: GRPC From 56aef679128cddd6d18624280a5bd48038778eed Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Mon, 28 Aug 2023 17:37:05 +0300 Subject: [PATCH 14/47] [heml] complete redesign of labels (#4788) --- k8s/charts/seaweedfs/templates/ca-cert.yaml | 5 +++ .../seaweedfs/templates/cert-caissuer.yaml | 5 +++ .../templates/cert-clusterissuer.yaml | 5 +++ .../seaweedfs/templates/client-cert.yaml | 5 +++ .../seaweedfs/templates/filer-cert.yaml | 6 +++ .../templates/filer-service-client.yaml | 11 +++-- .../seaweedfs/templates/filer-service.yaml | 11 +++-- .../templates/filer-servicemonitor.yaml | 11 +++-- .../templates/filer-statefulset.yaml | 27 ++++++----- .../{ingress.yaml => filler-ingress.yaml} | 45 +++---------------- .../seaweedfs/templates/master-cert.yaml | 6 +++ .../seaweedfs/templates/master-ingress.yaml | 42 +++++++++++++++++ .../seaweedfs/templates/master-service.yaml | 10 +++-- .../templates/master-servicemonitor.yaml | 7 ++- .../templates/master-statefulset.yaml | 26 ++++++----- .../seaweedfs/templates/s3-deployment.yaml | 24 +++++----- .../seaweedfs/templates/s3-service.yaml | 10 +++-- .../templates/s3-servicemonitor.yaml | 7 ++- .../templates/seaweedfs-s3-secret.yaml | 6 +++ .../templates/secret-seaweedfs-db.yaml | 5 +++ .../templates/security-configmap.yaml | 8 ++-- .../seaweedfs/templates/service-account.yaml | 15 +++++++ .../seaweedfs/templates/volume-cert.yaml | 6 +++ .../seaweedfs/templates/volume-service.yaml | 10 +++-- .../templates/volume-servicemonitor.yaml | 11 +++-- .../templates/volume-statefulset.yaml | 26 ++++++----- k8s/charts/seaweedfs/values.yaml | 1 - 27 files changed, 228 insertions(+), 123 deletions(-) rename k8s/charts/seaweedfs/templates/{ingress.yaml => filler-ingress.yaml} (50%) create mode 100644 k8s/charts/seaweedfs/templates/master-ingress.yaml diff --git a/k8s/charts/seaweedfs/templates/ca-cert.yaml b/k8s/charts/seaweedfs/templates/ca-cert.yaml index 9b74db0b1..c1d432d31 100644 --- a/k8s/charts/seaweedfs/templates/ca-cert.yaml +++ b/k8s/charts/seaweedfs/templates/ca-cert.yaml @@ -4,6 +4,11 @@ kind: Certificate metadata: name: {{ template "seaweedfs.name" . }}-ca-cert namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: secretName: {{ template "seaweedfs.name" . }}-ca-cert commonName: "{{ template "seaweedfs.name" . }}-root-ca" diff --git a/k8s/charts/seaweedfs/templates/cert-caissuer.yaml b/k8s/charts/seaweedfs/templates/cert-caissuer.yaml index 864652393..4c2693043 100644 --- a/k8s/charts/seaweedfs/templates/cert-caissuer.yaml +++ b/k8s/charts/seaweedfs/templates/cert-caissuer.yaml @@ -4,6 +4,11 @@ kind: Issuer metadata: name: {{ template "seaweedfs.name" . }}-ca-issuer namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: ca: secretName: {{ template "seaweedfs.name" . }}-ca-cert diff --git a/k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml b/k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml index 2604927db..c46aa4756 100644 --- a/k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml +++ b/k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml @@ -3,6 +3,11 @@ apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alph kind: ClusterIssuer metadata: name: {{ template "seaweedfs.name" . }}-clusterissuer + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: selfSigned: {} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/client-cert.yaml b/k8s/charts/seaweedfs/templates/client-cert.yaml index 7960c3f8d..5904a00b6 100644 --- a/k8s/charts/seaweedfs/templates/client-cert.yaml +++ b/k8s/charts/seaweedfs/templates/client-cert.yaml @@ -4,6 +4,11 @@ kind: Certificate metadata: name: {{ template "seaweedfs.name" . }}-client-cert namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: secretName: {{ template "seaweedfs.name" . }}-client-cert issuerRef: diff --git a/k8s/charts/seaweedfs/templates/filer-cert.yaml b/k8s/charts/seaweedfs/templates/filer-cert.yaml index 3c3565d33..8b5dbe257 100644 --- a/k8s/charts/seaweedfs/templates/filer-cert.yaml +++ b/k8s/charts/seaweedfs/templates/filer-cert.yaml @@ -4,6 +4,12 @@ kind: Certificate metadata: name: {{ template "seaweedfs.name" . }}-filer-cert namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer spec: secretName: {{ template "seaweedfs.name" . }}-filer-cert issuerRef: diff --git a/k8s/charts/seaweedfs/templates/filer-service-client.yaml b/k8s/charts/seaweedfs/templates/filer-service-client.yaml index 929b6f8bc..41251c897 100644 --- a/k8s/charts/seaweedfs/templates/filer-service-client.yaml +++ b/k8s/charts/seaweedfs/templates/filer-service-client.yaml @@ -4,8 +4,11 @@ metadata: name: {{ template "seaweedfs.name" . }}-filer-client namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer {{- if .Values.filer.metricsPort }} monitoring: "true" {{- end }} @@ -27,5 +30,5 @@ spec: protocol: TCP {{- end }} selector: - app: {{ template "seaweedfs.name" . }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: filer diff --git a/k8s/charts/seaweedfs/templates/filer-service.yaml b/k8s/charts/seaweedfs/templates/filer-service.yaml index ac44fa42b..352a20676 100644 --- a/k8s/charts/seaweedfs/templates/filer-service.yaml +++ b/k8s/charts/seaweedfs/templates/filer-service.yaml @@ -6,8 +6,11 @@ metadata: name: {{ template "seaweedfs.name" . }}-filer namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer spec: clusterIP: None publishNotReadyAddresses: true @@ -39,5 +42,5 @@ spec: protocol: TCP {{- end }} selector: - app: {{ template "seaweedfs.name" . }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: filer diff --git a/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml index 4a891463c..8035d7ffb 100644 --- a/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml @@ -6,8 +6,11 @@ metadata: name: {{ template "seaweedfs.name" . }}-filer namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer spec: endpoints: - interval: 30s @@ -15,7 +18,7 @@ spec: scrapeTimeout: 5s selector: matchLabels: - app: {{ template "seaweedfs.name" . }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: filer {{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index 7a2810a71..522a7dcfa 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -5,10 +5,11 @@ metadata: name: {{ template "seaweedfs.name" . }}-filer namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer spec: serviceName: {{ template "seaweedfs.name" . }}-filer podManagementPolicy: Parallel @@ -21,17 +22,17 @@ spec: {{- end }} selector: matchLabels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer template: metadata: labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer spec: restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }} {{- if .Values.filer.affinity }} @@ -180,8 +181,10 @@ spec: {{- end }} -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }} volumeMounts: + {{- if eq .Values.filer.logs.type "hostPath" }} - name: seaweedfs-filer-log-volume mountPath: "/logs/" + {{- end }} - mountPath: /etc/sw name: config-users readOnly: true diff --git a/k8s/charts/seaweedfs/templates/ingress.yaml b/k8s/charts/seaweedfs/templates/filler-ingress.yaml similarity index 50% rename from k8s/charts/seaweedfs/templates/ingress.yaml rename to k8s/charts/seaweedfs/templates/filler-ingress.yaml index 183cf1a23..b16fd7403 100644 --- a/k8s/charts/seaweedfs/templates/ingress.yaml +++ b/k8s/charts/seaweedfs/templates/filler-ingress.yaml @@ -12,6 +12,12 @@ metadata: namespace: {{ .Release.Namespace }} annotations: {{ omit .Values.filer.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer spec: ingressClassName: {{ .Values.filer.ingress.className | quote }} tls: @@ -35,41 +41,4 @@ spec: {{- if .Values.filer.ingress.host }} host: {{ .Values.filer.ingress.host }} {{- end }} -{{- end }} ---- -{{- if .Values.master.ingress.enabled }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }} -apiVersion: networking.k8s.io/v1beta1 -{{- else }} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: ingress-{{ template "seaweedfs.name" . }}-master - namespace: {{ .Release.Namespace }} - annotations: - {{ omit .Values.master.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }} -spec: - ingressClassName: {{ .Values.master.ingress.className | quote }} - rules: - - http: - paths: - - path: /sw-master/?(.*) - pathType: ImplementationSpecific - backend: -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} - service: - name: {{ template "seaweedfs.name" . }}-master - port: - number: {{ .Values.master.port }} - #name: -{{- else }} - serviceName: {{ template "seaweedfs.name" . }}-master - servicePort: {{ .Values.master.port }} -{{- end }} -{{- if .Values.filer.ingress.host }} - host: {{ .Values.master.ingress.host }} -{{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/k8s/charts/seaweedfs/templates/master-cert.yaml b/k8s/charts/seaweedfs/templates/master-cert.yaml index 1377ccda4..bb935eedb 100644 --- a/k8s/charts/seaweedfs/templates/master-cert.yaml +++ b/k8s/charts/seaweedfs/templates/master-cert.yaml @@ -4,6 +4,12 @@ kind: Certificate metadata: name: {{ template "seaweedfs.name" . }}-master-cert namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master spec: secretName: {{ template "seaweedfs.name" . }}-master-cert issuerRef: diff --git a/k8s/charts/seaweedfs/templates/master-ingress.yaml b/k8s/charts/seaweedfs/templates/master-ingress.yaml new file mode 100644 index 000000000..4522f0a06 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/master-ingress.yaml @@ -0,0 +1,42 @@ +{{- if .Values.master.ingress.enabled }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: networking.k8s.io/v1beta1 +{{- else }} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: ingress-{{ template "seaweedfs.name" . }}-master + namespace: {{ .Release.Namespace }} + annotations: + {{ omit .Values.master.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master +spec: + ingressClassName: {{ .Values.master.ingress.className | quote }} + rules: + - http: + paths: + - path: /sw-master/?(.*) + pathType: ImplementationSpecific + backend: +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} + service: + name: {{ template "seaweedfs.name" . }}-master + port: + number: {{ .Values.master.port }} + #name: +{{- else }} + serviceName: {{ template "seaweedfs.name" . }}-master + servicePort: {{ .Values.master.port }} +{{- end }} +{{- if .Values.filer.ingress.host }} + host: {{ .Values.master.ingress.host }} +{{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/master-service.yaml b/k8s/charts/seaweedfs/templates/master-service.yaml index 77710220b..978f9c881 100644 --- a/k8s/charts/seaweedfs/templates/master-service.yaml +++ b/k8s/charts/seaweedfs/templates/master-service.yaml @@ -4,8 +4,10 @@ metadata: name: {{ template "seaweedfs.name" . }}-master namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: master + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: master + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} annotations: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" spec: @@ -27,5 +29,5 @@ spec: protocol: TCP {{- end }} selector: - app: {{ template "seaweedfs.name" . }} - component: master + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: master diff --git a/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml index 64ee18035..0332f756d 100644 --- a/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml @@ -6,8 +6,11 @@ metadata: name: {{ template "seaweedfs.name" . }}-master namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: master + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master spec: endpoints: - interval: 30s diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index e1b272ab4..2b4a8b5c9 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -5,10 +5,10 @@ metadata: name: {{ template "seaweedfs.name" . }}-master namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: serviceName: {{ template "seaweedfs.name" . }}-master podManagementPolicy: Parallel @@ -21,17 +21,17 @@ spec: {{- end }} selector: matchLabels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: master + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master template: metadata: labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: master + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master spec: restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }} {{- if .Values.master.affinity }} @@ -135,8 +135,10 @@ spec: volumeMounts: - name : data-{{ .Release.Namespace }} mountPath: /data + {{- if eq .Values.master.logs.type "hostPath" }} - name: seaweedfs-master-log-volume mountPath: "/logs/" + {{- end }} {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index 2040e7161..a996b6ef6 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -5,25 +5,25 @@ metadata: name: {{ template "seaweedfs.name" . }}-s3 namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: replicas: {{ .Values.s3.replicas }} selector: matchLabels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: s3 + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: s3 template: metadata: labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: s3 + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: s3 spec: restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }} {{- if .Values.s3.tolerations }} diff --git a/k8s/charts/seaweedfs/templates/s3-service.yaml b/k8s/charts/seaweedfs/templates/s3-service.yaml index 133ce3372..32fef5d81 100644 --- a/k8s/charts/seaweedfs/templates/s3-service.yaml +++ b/k8s/charts/seaweedfs/templates/s3-service.yaml @@ -4,8 +4,10 @@ metadata: name: {{ template "seaweedfs.name" . }}-s3 namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: s3 + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: s3 + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} spec: ports: - name: "swfs-s3" @@ -25,5 +27,5 @@ spec: protocol: TCP {{- end }} selector: - app: {{ template "seaweedfs.name" . }} - component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }} + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }} diff --git a/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml index 5df537086..876f72dcd 100644 --- a/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml @@ -6,8 +6,11 @@ metadata: name: {{ template "seaweedfs.name" . }}-s3 namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: s3 + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: s3 spec: endpoints: - interval: 30s diff --git a/k8s/charts/seaweedfs/templates/seaweedfs-s3-secret.yaml b/k8s/charts/seaweedfs/templates/seaweedfs-s3-secret.yaml index 4e9189633..0f3674380 100644 --- a/k8s/charts/seaweedfs/templates/seaweedfs-s3-secret.yaml +++ b/k8s/charts/seaweedfs/templates/seaweedfs-s3-secret.yaml @@ -12,6 +12,12 @@ metadata: annotations: "helm.sh/resource-policy": keep "helm.sh/hook": "pre-install" + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: s3 stringData: admin_access_key_id: {{ $access_key_admin }} admin_secret_access_key: {{ $secret_key_admin }} diff --git a/k8s/charts/seaweedfs/templates/secret-seaweedfs-db.yaml b/k8s/charts/seaweedfs/templates/secret-seaweedfs-db.yaml index c6132c9ea..9a1aca91d 100644 --- a/k8s/charts/seaweedfs/templates/secret-seaweedfs-db.yaml +++ b/k8s/charts/seaweedfs/templates/secret-seaweedfs-db.yaml @@ -7,6 +7,11 @@ metadata: annotations: "helm.sh/resource-policy": keep "helm.sh/hook": "pre-install" + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} stringData: user: "YourSWUser" password: "HardCodedPassword" diff --git a/k8s/charts/seaweedfs/templates/security-configmap.yaml b/k8s/charts/seaweedfs/templates/security-configmap.yaml index 7d06614ec..8f82c25a9 100644 --- a/k8s/charts/seaweedfs/templates/security-configmap.yaml +++ b/k8s/charts/seaweedfs/templates/security-configmap.yaml @@ -5,10 +5,10 @@ metadata: name: {{ template "seaweedfs.name" . }}-security-config namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} data: security.toml: |- # this file is read by master, volume server, and filer diff --git a/k8s/charts/seaweedfs/templates/service-account.yaml b/k8s/charts/seaweedfs/templates/service-account.yaml index 89f531174..a0ca3f004 100644 --- a/k8s/charts/seaweedfs/templates/service-account.yaml +++ b/k8s/charts/seaweedfs/templates/service-account.yaml @@ -5,6 +5,11 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: seaweedfs-rw-cr + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} rules: - apiGroups: [""] resources: ["pods"] @@ -15,11 +20,21 @@ kind: ServiceAccount metadata: name: seaweedfs-rw-sa namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:serviceaccount:seaweedfs-rw-sa:default + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} subjects: - kind: ServiceAccount name: seaweedfs-rw-sa diff --git a/k8s/charts/seaweedfs/templates/volume-cert.yaml b/k8s/charts/seaweedfs/templates/volume-cert.yaml index 4167867b1..c6b333f7c 100644 --- a/k8s/charts/seaweedfs/templates/volume-cert.yaml +++ b/k8s/charts/seaweedfs/templates/volume-cert.yaml @@ -4,6 +4,12 @@ kind: Certificate metadata: name: {{ template "seaweedfs.name" . }}-volume-cert namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume spec: secretName: {{ template "seaweedfs.name" . }}-volume-cert issuerRef: diff --git a/k8s/charts/seaweedfs/templates/volume-service.yaml b/k8s/charts/seaweedfs/templates/volume-service.yaml index 1e07865c0..20539c5a1 100644 --- a/k8s/charts/seaweedfs/templates/volume-service.yaml +++ b/k8s/charts/seaweedfs/templates/volume-service.yaml @@ -4,8 +4,10 @@ metadata: name: {{ template "seaweedfs.name" . }}-volume namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: volume + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: volume + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} spec: clusterIP: None ports: @@ -24,5 +26,5 @@ spec: protocol: TCP {{- end }} selector: - app: {{ template "seaweedfs.name" . }} - component: volume \ No newline at end of file + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: volume \ No newline at end of file diff --git a/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml index 2dcca5ac8..d2ee8b4c7 100644 --- a/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml @@ -6,8 +6,11 @@ metadata: name: {{ template "seaweedfs.name" . }}-volume namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - component: volume + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume spec: endpoints: - interval: 30s @@ -15,7 +18,7 @@ spec: scrapeTimeout: 5s selector: matchLabels: - app: {{ template "seaweedfs.name" . }} - component: volume + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/component: volume {{- end }} {{- end }} \ No newline at end of file diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 4abadb24d..18b342dca 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -5,27 +5,27 @@ metadata: name: {{ template "seaweedfs.name" . }}-volume namespace: {{ .Release.Namespace }} labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} spec: serviceName: {{ template "seaweedfs.name" . }}-volume replicas: {{ .Values.volume.replicas }} podManagementPolicy: Parallel selector: matchLabels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: volume + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume template: metadata: labels: - app: {{ template "seaweedfs.name" . }} - chart: {{ template "seaweedfs.chart" . }} - release: {{ .Release.Name }} - component: volume + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume spec: {{- if .Values.volume.affinity }} affinity: @@ -145,8 +145,10 @@ spec: - name: idx mountPath: "{{ .Values.volume.dir_idx }}/" {{- end }} + {{- if eq .Values.volume.logs.type "hostPath" }} - name: logs mountPath: "/logs/" + {{- end }} {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 165be23f3..8431047b9 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -265,7 +265,6 @@ volume: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" - filer: enabled: true repository: null From 47fae7d0ebf13266011bceacf679e25088d8113c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 08:33:51 -0700 Subject: [PATCH 15/47] Bump docker/setup-buildx-action from 2.9.1 to 2.10.0 (#4789) --- .github/workflows/container_dev.yml | 2 +- .github/workflows/container_latest.yml | 2 +- .github/workflows/container_release1.yml | 2 +- .github/workflows/container_release2.yml | 2 +- .github/workflows/container_release3.yml | 2 +- .github/workflows/container_release4.yml | 2 +- .github/workflows/container_release5.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 12fbb7599..d0591455a 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -36,7 +36,7 @@ jobs: uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 194511219..b7c8b650b 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index 5de2562e5..b88871d30 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index a201551b7..2a9f68404 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 18a9a1f59..b4ba92bf6 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 3472bf394..0526eb842 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index ea17f5592..17d36862f 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' From 67f1bb69c434e21a03a06a391f2315d3a2e2af8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:04:10 -0700 Subject: [PATCH 16/47] Bump github.com/jinzhu/copier from 0.3.5 to 0.4.0 (#4791) Bumps [github.com/jinzhu/copier](https://github.com/jinzhu/copier) from 0.3.5 to 0.4.0. - [Commits](https://github.com/jinzhu/copier/compare/v0.3.5...v0.4.0) --- updated-dependencies: - dependency-name: github.com/jinzhu/copier dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 11 +++++------ go.sum | 8 ++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index eb4ef3c5b..cb7876967 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gocql/gocql v1.5.2 - github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 github.com/golang/snappy v0.0.4 // indirect @@ -53,7 +53,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect - github.com/jinzhu/copier v0.3.5 + github.com/jinzhu/copier v0.4.0 github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 github.com/karlseguin/ccache/v2 v2.0.8 @@ -131,7 +131,7 @@ require ( modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect modernc.org/libc v1.22.5 // indirect - modernc.org/mathutil v1.5.0 // indirect + modernc.org/mathutil v1.5.0 modernc.org/memory v1.5.0 // indirect modernc.org/opt v0.1.3 // indirect modernc.org/sqlite v1.24.0 @@ -148,10 +148,12 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.34 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 github.com/fluent/fluent-logger-golang v1.9.0 + github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50 github.com/hanwen/go-fuse/v2 v2.3.0 github.com/hashicorp/raft v1.5.0 github.com/hashicorp/raft-boltdb/v2 v2.2.2 + github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/puzpuzpuz/xsync/v2 v2.4.1 github.com/rabbitmq/amqp091-go v1.8.1 github.com/rclone/rclone v1.63.1 @@ -212,7 +214,6 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect @@ -250,8 +251,6 @@ require ( github.com/ncw/swift/v2 v2.0.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/oracle/oci-go-sdk/v65 v65.34.0 // indirect - github.com/orcaman/concurrent-map v1.0.0 // indirect - github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect diff --git a/go.sum b/go.sum index 0fdaef931..80d08921c 100644 --- a/go.sum +++ b/go.sum @@ -503,8 +503,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= -github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -650,12 +650,8 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/oracle/oci-go-sdk/v65 v65.34.0 h1:uG1KucBxAbn8cYRgQHxtQKogtl85nOX8LhimZCPfMqw= github.com/oracle/oci-go-sdk/v65 v65.34.0/go.mod h1:MXMLMzHnnd9wlpgadPkdlkZ9YrwQmCOmbX5kjVEJodw= -github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= -github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= -github.com/ovh/go-ovh v1.4.1/go.mod h1:6bL6pPyUT7tBfI0pqOegJgRjgjuO+mOo+MyXd1EEC0M= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= From ac1b1445260b130c171d60ce9ea13279522a5a87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:04:59 -0700 Subject: [PATCH 17/47] Bump gocloud.dev from 0.33.0 to 0.34.0 (#4795) Bumps [gocloud.dev](https://github.com/google/go-cloud) from 0.33.0 to 0.34.0. - [Release notes](https://github.com/google/go-cloud/releases) - [Commits](https://github.com/google/go-cloud/compare/v0.33.0...v0.34.0) --- updated-dependencies: - dependency-name: gocloud.dev dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cb7876967..edb4590d3 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( go.etcd.io/etcd/client/v3 v3.5.9 go.mongodb.org/mongo-driver v1.12.1 go.opencensus.io v0.24.0 // indirect - gocloud.dev v0.33.0 + gocloud.dev v0.34.0 gocloud.dev/pubsub/natspubsub v0.33.0 gocloud.dev/pubsub/rabbitpubsub v0.33.0 golang.org/x/crypto v0.12.0 // indirect diff --git a/go.sum b/go.sum index 80d08921c..7138dfd36 100644 --- a/go.sum +++ b/go.sum @@ -934,8 +934,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -gocloud.dev v0.33.0 h1:ET5z49jm1+eUhY5BkuGk2d7czfgGeXKd4vtg1Jcg9OQ= -gocloud.dev v0.33.0/go.mod h1:z6W8qorjrfM09H8t1MDk8KLPj3Xi26aFBzDKAHWIgLU= +gocloud.dev v0.34.0 h1:LzlQY+4l2cMtuNfwT2ht4+fiXwWf/NmPTnXUlLmGif4= +gocloud.dev v0.34.0/go.mod h1:psKOachbnvY3DAOPbsFVmLIErwsbWPUG2H5i65D38vE= gocloud.dev/pubsub/natspubsub v0.33.0 h1:SwHh+PuDmZ3/xv1h/7VBRVlgiBI9Hw6vS7NcvJzW3Us= gocloud.dev/pubsub/natspubsub v0.33.0/go.mod h1:2hKJ2+CbUyuSv4pFkaU/Ay3buK5ACJMxqlfnVSXFZGs= gocloud.dev/pubsub/rabbitpubsub v0.33.0 h1:zidscDuR/INRVV/MKepaVbx/XIhgGKSCosiqfhORDZE= From db5d23defe49337c05ec7b7c2b793ba9be3616c0 Mon Sep 17 00:00:00 2001 From: Alexandre Viau Date: Mon, 28 Aug 2023 12:06:06 -0400 Subject: [PATCH 18/47] k8s: allow configuring serviceAccountName (#4790) --- .../seaweedfs/templates/filer-statefulset.yaml | 3 +++ .../seaweedfs/templates/master-statefulset.yaml | 3 +++ .../seaweedfs/templates/s3-deployment.yaml | 3 +++ .../seaweedfs/templates/volume-statefulset.yaml | 3 +++ k8s/charts/seaweedfs/values.yaml | 16 ++++++++++++++++ 5 files changed, 28 insertions(+) diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index 522a7dcfa..48279dad7 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -52,6 +52,9 @@ spec: priorityClassName: {{ .Values.filer.priorityClassName | quote }} {{- end }} enableServiceLinks: false + {{- if .Values.filer.serviceAccountName }} + serviceAccountName: {{ .Values.filer.serviceAccountName | quote }} + {{- end }} {{- if .Values.filer.initContainers }} initContainers: {{ tpl .Values.filer.initContainers . | nindent 8 | trim }} diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index 2b4a8b5c9..e8563ff9b 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -48,6 +48,9 @@ spec: priorityClassName: {{ .Values.master.priorityClassName | quote }} {{- end }} enableServiceLinks: false + {{- if .Values.master.serviceAccountName }} + serviceAccountName: {{ .Values.master.serviceAccountName | quote }} + {{- end }} {{- if .Values.master.initContainers }} initContainers: {{ tpl .Values.master.initContainers . | nindent 8 | trim }} diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index a996b6ef6..2fe3fee92 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -36,6 +36,9 @@ spec: priorityClassName: {{ .Values.s3.priorityClassName | quote }} {{- end }} enableServiceLinks: false + {{- if .Values.s3.serviceAccountName }} + serviceAccountName: {{ .Values.s3.serviceAccountName | quote }} + {{- end }} {{- if .Values.s3.initContainers }} initContainers: {{ tpl .Values.s3.initContainers . | nindent 8 | trim }} diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 18b342dca..344821a4d 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -42,6 +42,9 @@ spec: priorityClassName: {{ .Values.volume.priorityClassName | quote }} {{- end }} enableServiceLinks: false + {{- if .Values.volume.serviceAccountName }} + serviceAccountName: {{ .Values.volume.serviceAccountName | quote }} + {{- end }} {{- $initContainers_exists := include "volume.initContainers_exists" . -}} {{- if $initContainers_exists }} initContainers: diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 8431047b9..1f24d221c 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -124,6 +124,10 @@ master: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + # used to assign a service account. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + serviceAccountName: "" + ingress: enabled: false className: "nginx" @@ -265,6 +269,10 @@ volume: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + # used to assign a service account. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + serviceAccountName: "" + filer: enabled: true repository: null @@ -372,6 +380,10 @@ filer: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + # used to assign a service account. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + serviceAccountName: "" + ingress: enabled: false className: "nginx" @@ -492,6 +504,10 @@ s3: # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ priorityClassName: "" + # used to assign a service account. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + serviceAccountName: "" + logs: type: "hostPath" size: "" From cf44633491b4d7fd1ab1109d78ff503346e96c09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:18:22 -0700 Subject: [PATCH 19/47] Bump github.com/aws/aws-sdk-go-v2/credentials from 1.13.34 to 1.13.35 (#4792) Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.13.34 to 1.13.35. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.13.34...credentials/v1.13.35) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index edb4590d3..4424f0849 100644 --- a/go.mod +++ b/go.mod @@ -143,9 +143,9 @@ require ( github.com/Jille/raft-grpc-transport v1.4.0 github.com/arangodb/go-driver v1.6.0 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go-v2 v1.20.3 + github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.33 - github.com/aws/aws-sdk-go-v2/credentials v1.13.34 + github.com/aws/aws-sdk-go-v2/credentials v1.13.35 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 github.com/fluent/fluent-logger-golang v1.9.0 github.com/golang-jwt/jwt/v5 v5.0.0 @@ -180,20 +180,20 @@ require ( github.com/abbot/go-http-auth v0.4.0 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.40 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.34 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.21.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect github.com/aws/smithy-go v1.14.2 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/boltdb/bolt v1.3.1 // indirect diff --git a/go.sum b/go.sum index 7138dfd36..2b6b71a1c 100644 --- a/go.sum +++ b/go.sum @@ -115,26 +115,26 @@ github.com/aws/aws-sdk-go v1.44.314 h1:d/5Jyk/Fb+PBd/4nzQg0JuC2W4A0knrDIzBgK/ggA github.com/aws/aws-sdk-go v1.44.314/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4= github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= -github.com/aws/aws-sdk-go-v2 v1.20.3 h1:lgeKmAZhlj1JqN43bogrM75spIvYnRxqTAh1iupu1yE= -github.com/aws/aws-sdk-go-v2 v1.20.3/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 h1:lN6L3LrYHeZ6xCxaIYtoWCx4GMLk4nRknsh29OMSqHY= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12/go.mod h1:TDCkEAkMTXxTs0oLBGBKpBZbk3NLh8EvAfF0Q3x8/0c= github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= -github.com/aws/aws-sdk-go-v2/credentials v1.13.34 h1:/EYG4lzayDd5PY6HQQ2Qyj/cD6CR3kz96BjTZAO5tNo= -github.com/aws/aws-sdk-go-v2/credentials v1.13.34/go.mod h1:+wgdxCGNulHme6kTMZuDL9KOagLPloemoYkfjpQkSEU= +github.com/aws/aws-sdk-go-v2/credentials v1.13.35 h1:QpsNitYJu0GgvMBLUIYu9H4yryA5kMksjeIVQfgXrt8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.35/go.mod h1:o7rCaLtvK0hUggAGclf76mNGGkaG5a9KWlp+d9IpcV8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.10 h1:mgOrtwYfJZ4e3QJe1TrliC/xIkauafGMdLLuCExOqcs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.10/go.mod h1:wMsSLVM2hRpDVhd+3dtLUzqwm7/fjuhNN+b1aOLDt6g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.40 h1:CXceCS9BrDInRc74GDCQ8Qyk/Gp9VLdK+Rlve+zELSE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.40/go.mod h1:5kKmFhLeOVy6pwPDpDNA6/hK/d6URC98pqDDqHgdBx4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.34 h1:B+nZtd22cbko5+793hg7LEaTeLMiZwlgCLUrN5Y0uzg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.34/go.mod h1:RZP0scceAyhMIQ9JvFp7HvkpcgqjL4l/4C+7RAeGbuM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39/go.mod h1:WLAW8PT7+JhjZfLSWe7WEJaJu0GNo0cKc2Zyo003RBs= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 h1:vUh7dBFNS3oFCtVv6CiYKh5hP9ls8+kIpKLeFruIBLk= @@ -144,8 +144,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13/go.mod h1: github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 h1:QviNkc+vGSuEHx8P+pVNKOdWLXBPIwMFv7p0fphgE4U= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33/go.mod h1:fABTUmOrAgAalG2i9WJpjBvlnk7UK8YmnYaxN+Q2CwE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32/go.mod h1:4jwAWKEkCR0anWk5+1RbfSg1R5Gzld7NLiuaq5bTR/Y= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.34 h1:JwvXk+1ePAD9xkFHprhHYqwsxLDcbNFsPI1IAT2sPS0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.34/go.mod h1:ytsF+t+FApY2lFnN51fJKPhH6ICKOPXKEcwwgmJEdWI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 h1:PT6PBCycRwhpEW5hJnRiceCeoWJ+r3bdgXtV+VKG7Pk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1/go.mod h1:TqoxCLwT2nrxrBGA+z7t6OWM7LBkgRckK3gOjYE+7JA= github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 h1:v346f1h8sUBKXnEbrv43L37MTBlFHyKXQPIZHNAaghA= @@ -155,14 +155,14 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.21.1/go.mod h1:laHbYFVzphXdCiT3gitfu github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 h1:KbGaxApdPOT2ZWqJiQY5ApnpNhUGbGTjYiKAidlFwp8= github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1/go.mod h1:+phkm4aFvcM4jbsDRGoZ+mD8MMvksHF459Xpy5Z90f0= github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.4 h1:WZPZ7Zf6Yo13lsfTetFrLU/7hZ9CXESDpdIHvmLxQFQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.4/go.mod h1:FP05hDXTLouXwAMQ1swqybHy7tHySblMkBMKSumaKg0= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 h1:oCvTFSDi67AX0pOX3PuPdGFewvLRU2zzFSrTsgURNo0= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.5/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.4 h1:pYFM2U/3/4RLrlMSYXwL1XPBCWvaePk2p+0+i/BgHOs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.4/go.mod h1:4pdlNASc29u0j9bq2jIQcBghG5Lx2oQAIj91vo1u1t8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 h1:dnInJb4S0oy8aQuri1mV6ipLlnZPfnsDNB9BGO9PDNY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.4 h1:zj4jxK3L54tGyqKleKDMK4vHolENxlq11dF0v1oBkJo= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.4/go.mod h1:CQRMCzYvl5eeAQW3AWkRLS+zGGXCucBnsiQlrs+tCeo= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= From 1faf9afb9190f2e62aadbb79246ff2d60ca4df54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:18:35 -0700 Subject: [PATCH 20/47] Bump modernc.org/sqlite from 1.24.0 to 1.25.0 (#4793) Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.24.0 to 1.25.0. - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.24.0...v1.25.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 4424f0849..f62cd539f 100644 --- a/go.mod +++ b/go.mod @@ -130,11 +130,11 @@ require ( modernc.org/b v1.0.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect - modernc.org/libc v1.22.5 // indirect + modernc.org/libc v1.24.1 // indirect modernc.org/mathutil v1.5.0 - modernc.org/memory v1.5.0 // indirect + modernc.org/memory v1.6.0 // indirect modernc.org/opt v0.1.3 // indirect - modernc.org/sqlite v1.24.0 + modernc.org/sqlite v1.25.0 modernc.org/strutil v1.1.3 modernc.org/token v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2b6b71a1c..a6a706371 100644 --- a/go.sum +++ b/go.sum @@ -1422,17 +1422,17 @@ modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= -modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= +modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= +modernc.org/memory v1.6.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.24.0 h1:EsClRIWHGhLTCX44p+Ri/JLD+vFGo0QGjasg2/F9TlI= -modernc.org/sqlite v1.24.0/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= +modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= +modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= From 03389bf8441d6130bc36b4a1732e5dc12fb8fd2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:18:44 -0700 Subject: [PATCH 21/47] Bump github.com/gocql/gocql from 1.5.2 to 1.6.0 (#4794) Bumps [github.com/gocql/gocql](https://github.com/gocql/gocql) from 1.5.2 to 1.6.0. - [Release notes](https://github.com/gocql/gocql/releases) - [Changelog](https://github.com/gocql/gocql/blob/master/CHANGELOG.md) - [Commits](https://github.com/gocql/gocql/compare/v1.5.2...v1.6.0) --- updated-dependencies: - dependency-name: github.com/gocql/gocql dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f62cd539f..f0d056ebb 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/go-redsync/redsync/v4 v4.8.1 github.com/go-sql-driver/mysql v1.7.1 github.com/go-zookeeper/zk v1.0.3 // indirect - github.com/gocql/gocql v1.5.2 + github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 diff --git a/go.sum b/go.sum index a6a706371..93675551b 100644 --- a/go.sum +++ b/go.sum @@ -318,8 +318,8 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= -github.com/gocql/gocql v1.5.2 h1:WnKf8xRQImcT/KLaEWG2pjEeryDB7K0qQN9mPs1C58Q= -github.com/gocql/gocql v1.5.2/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= +github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= +github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= From 271c51f273f368c01ef2ed14bc2d8966e63cccdb Mon Sep 17 00:00:00 2001 From: Alexandre Viau Date: Mon, 28 Aug 2023 23:44:09 -0400 Subject: [PATCH 22/47] helm: allow setting master.toml config (#4797) --- .../seaweedfs/templates/master-configmap.yaml | 15 +++++++++++++++ .../seaweedfs/templates/master-statefulset.yaml | 7 +++++++ k8s/charts/seaweedfs/values.yaml | 4 ++++ 3 files changed, 26 insertions(+) create mode 100644 k8s/charts/seaweedfs/templates/master-configmap.yaml diff --git a/k8s/charts/seaweedfs/templates/master-configmap.yaml b/k8s/charts/seaweedfs/templates/master-configmap.yaml new file mode 100644 index 000000000..73155e87d --- /dev/null +++ b/k8s/charts/seaweedfs/templates/master-configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.master.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "seaweedfs.name" . }}-master-config + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +data: + master.toml: |- + {{ .Values.master.config | nindent 4 }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index e8563ff9b..ba4b54c27 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -142,6 +142,10 @@ spec: - name: seaweedfs-master-log-volume mountPath: "/logs/" {{- end }} + - name: master-config + readOnly: true + mountPath: /etc/seaweedfs/master.toml + subPath: master.toml {{- if .Values.global.enableSecurity }} - name: security-config readOnly: true @@ -212,6 +216,9 @@ spec: path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/ type: DirectoryOrCreate {{- end }} + - name: master-config + configMap: + name: {{ template "seaweedfs.name" . }}-master-config {{- if .Values.global.enableSecurity }} - name: security-config configMap: diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 1f24d221c..af73cda93 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -62,6 +62,10 @@ master: # Disable http request, only gRpc operations are allowed disableHttp: false + config: |- + # Enter any extra configuration for master.toml here. + # It may be be a multi-line string. + # can use ANY storage-class , example with local-path-provisioner # data: # type: "persistentVolumeClaim" From de1dcf3b2f0b7cf6b62304a7b7835e876936dbdf Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Tue, 29 Aug 2023 18:43:11 +0300 Subject: [PATCH 23/47] [heml] fix default affinity and allow owerride ingress annotations (#4799) --- .../seaweedfs/templates/filler-ingress.yaml | 4 +++- .../seaweedfs/templates/master-ingress.yaml | 4 +++- k8s/charts/seaweedfs/values.yaml | 22 +++++++++---------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/k8s/charts/seaweedfs/templates/filler-ingress.yaml b/k8s/charts/seaweedfs/templates/filler-ingress.yaml index b16fd7403..5580bb188 100644 --- a/k8s/charts/seaweedfs/templates/filler-ingress.yaml +++ b/k8s/charts/seaweedfs/templates/filler-ingress.yaml @@ -10,8 +10,10 @@ kind: Ingress metadata: name: ingress-{{ template "seaweedfs.name" . }}-filer namespace: {{ .Release.Namespace }} + {{- if .Values.filer.ingress.annotations }} annotations: - {{ omit .Values.filer.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }} + {{ tpl .Values.filer.ingress.annotations . | nindent 4 | trim }} + {{- end }} labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} diff --git a/k8s/charts/seaweedfs/templates/master-ingress.yaml b/k8s/charts/seaweedfs/templates/master-ingress.yaml index 4522f0a06..cce4c1d8f 100644 --- a/k8s/charts/seaweedfs/templates/master-ingress.yaml +++ b/k8s/charts/seaweedfs/templates/master-ingress.yaml @@ -10,8 +10,10 @@ kind: Ingress metadata: name: ingress-{{ template "seaweedfs.name" . }}-master namespace: {{ .Release.Namespace }} + {{- if .Values.master.ingress.annotations }} annotations: - {{ omit .Values.master.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }} + {{ tpl .Values.master.ingress.annotations . | nindent 4 | trim }} + {{- end }} labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index af73cda93..786200bec 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -106,9 +106,9 @@ master: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: - app: {{ template "seaweedfs.name" . }} - release: "{{ .Release.Name }}" - component: master + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master topologyKey: kubernetes.io/hostname # Toleration Settings for master pods @@ -137,7 +137,7 @@ master: className: "nginx" # host: false for "*" hostname host: "master.seaweedfs.local" - annotations: + annotations: | nginx.ingress.kubernetes.io/auth-type: "basic" nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret" nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master' @@ -245,9 +245,9 @@ volume: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: - app: {{ template "seaweedfs.name" . }} - release: "{{ .Release.Name }}" - component: volume + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume topologyKey: kubernetes.io/hostname # Resource requests, limits, etc. for the server cluster placement. This @@ -352,9 +352,9 @@ filer: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: - app: {{ template "seaweedfs.name" . }} - release: "{{ .Release.Name }}" - component: filer + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer topologyKey: kubernetes.io/hostname # updatePartition is used to control a careful rolling update of SeaweedFS @@ -393,7 +393,7 @@ filer: className: "nginx" # host: false for "*" hostname host: "seaweedfs.cluster.local" - annotations: + annotations: | nginx.ingress.kubernetes.io/backend-protocol: GRPC nginx.ingress.kubernetes.io/auth-type: "basic" nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret" From f139a56cb95ddf9823226bc2d9fbdad31cb69511 Mon Sep 17 00:00:00 2001 From: ginn13 Date: Thu, 31 Aug 2023 16:42:07 +0300 Subject: [PATCH 24/47] Add master servers to grafana dashboard (#4801) --- .../seaweedfs-grafana-dashboard.json | 4522 ++++++++++------- 1 file changed, 2769 insertions(+), 1753 deletions(-) diff --git a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json index 3e5621e4e..da0a4973b 100644 --- a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json +++ b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json @@ -1,30 +1,11 @@ { - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "4.6.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], "annotations": { "list": [ { "builtIn": 1, - "datasource": "${DS_PROMETHEUS}", + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -36,1789 +17,2824 @@ ] }, "editable": true, + "fiscalYearStartMonth": 0, "gnetId": 10423, "graphTooltip": 0, - "hideControls": false, - "id": null, + "id": 1360, + "iteration": 1693482389821, "links": [], - "refresh": "30s", - "rows": [ + "liveNow": false, + "panels": [ { - "collapse": true, - "height": 251, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 67, + "panels": [], + "title": "Master", + "type": "row" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Whether master is leader or not", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Request Duration 90th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "unit": "bool_yes_no" }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 49, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 - }, - { - "expr": "", - "format": "time_series", - "intervalFactor": 2, - "refId": "C" - } + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 57, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Request Duration 95th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "fields": "", + "values": false }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.15", + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 45, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 - }, - { - "expr": "", - "format": "time_series", - "intervalFactor": 2, - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Request Duration 99th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(SeaweedFS_filer_request_total[1m]) * 5", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 30 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer QPS", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "exemplar": true, + "expr": "sum by (pod) (SeaweedFS_master_is_leader{job=\"seaweedfs-master\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 60 } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Raft leader", + "type": "stat" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Count times leader changed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 68, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum by (pod) (SeaweedFS_master_leader_changes{job=\"seaweedfs-master\", type=~\".+\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 60 + } + ], + "title": "Master leader changes", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Heartbeats received from components", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 69, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum by (type) (increase(SeaweedFS_master_received_heartbeats{job=\"seaweedfs-master\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 60 + } + ], + "title": "Received heartbeats", + "type": "timeseries" + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "message": "", + "name": "Replica Placement Mismatch alert", + "noDataState": "ok", + "notifications": [] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Count replica placement mismatch", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 70, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum (SeaweedFS_master_replica_placement_mismatch{job=\"seaweedfs-master\"} > 0) by (pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "gt", + "value": 0, + "visible": true + } + ], + "title": "Replica Placement Mismatch", + "type": "timeseries" + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ + { + "evaluator": { + "params": [ + 1, + 1 + ], + "type": "outside_range" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "message": "Raft leader count of master-servers not equal to 1", + "name": "Raft leader alert", + "noDataState": "no_data", + "notifications": [] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total count of raft leaders", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 71, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "exemplar": true, + "expr": "sum (SeaweedFS_master_is_leader{job=\"seaweedfs-master\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Leaders", + "refId": "A", + "step": 60 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "lt", + "value": 1, + "visible": true + }, + { + "colorMode": "critical", + "op": "gt", + "value": 1, + "visible": true + } + ], + "title": "Raft leader count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Whether cluster locked or not", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 74, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.5.15", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "SeaweedFS_master_admin_lock{job=\"seaweedfs-master\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Client IP: {{client}}", + "range": true, + "refId": "A", + "step": 60 + } + ], + "title": "Admin lock", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 60, + "panels": [], "title": "Filer", - "titleSize": "h6" + "type": "row" }, { - "collapse": false, - "height": 250, - "panels": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 56, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "S3 Request Duration 90th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 57, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "S3 Request Duration 95th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 58, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 60 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "S3 Request Duration 99th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 55, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(SeaweedFS_s3_request_total[1m]) * 5", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 30 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "S3 API QPS", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "hideTimeOverride": false, - "id": 59, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false - } - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (type) (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}} requests", - "refId": "A", - "step": 30 - }, - { - "expr": "sum (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "All PUT, COPY, POST, LIST", - "refId": "C", - "step": 30 - }, - { - "expr": "sum (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "GET and all other", - "refId": "B" - }, - { - "expr": "sum by (type) (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}} requests", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": "1M", - "timeShift": null, - "title": "S3 API Monthly Cost if on AWS", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "currencyUSD", - "label": "Cost in US$", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "currencyUSD", - "label": "Write Cost", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "thresholds": [], + "timeRegions": [], + "title": "Filer Request Duration 90th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 2, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filer Request Duration 95th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 2, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filer Request Duration 99th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:810", + "alias": "total", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (type) (rate(SeaweedFS_filer_request_total[$__rate_interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filer QPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:817", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:818", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 61, + "panels": [], "title": "S3 Gateway", - "titleSize": "h6" + "type": "row" }, { - "collapse": true, - "height": 252, - "panels": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 47, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[1m])) by (le, exported_instance))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{exported_instance}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "average", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Volume Server Request Duration 99th percentile", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "total", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(SeaweedFS_volumeServer_request_total[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Volume Server QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "id": 48, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(SeaweedFS_volumeServer_volumes) by (collection, type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{collection}} {{type}}", - "refId": "A" - }, - { - "expr": "sum(SeaweedFS_volumeServer_max_volumes)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Volume Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (collection, type)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{collection}} {{type}}", - "refId": "A" - }, - { - "expr": "sum(SeaweedFS_volumeServer_total_disk_size)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Used Disk Space by Collection and Type", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "id": 51, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (exported_instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{exported_instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Used Disk Space by Host", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "thresholds": [], + "timeRegions": [], + "title": "S3 Request Duration 90th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 29 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 Request Duration 95th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 29 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 Request Duration 99th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:115", + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:116", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 72, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type, pod))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{pod}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 Request. Duration 99th percentile per instance", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:115", + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:116", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 43 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type, bucket))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}-{{bucket}}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 Request Duration 99th percentile per bucket", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:115", + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:116", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 50 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:689", + "alias": "total", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum (rate(SeaweedFS_s3_request_total[$__rate_interval])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "S3 API QPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:696", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:697", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (type) (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}} requests", + "refId": "A", + "step": 30 + }, + { + "expr": "sum (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "All PUT, COPY, POST, LIST", + "refId": "C", + "step": 30 + }, + { + "expr": "sum (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "GET and all other", + "refId": "B" + }, + { + "expr": "sum by (type) (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{type}} requests", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": "1M", + "timeRegions": [], + "title": "S3 API Monthly Cost if on AWS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "label": "Cost in US$", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "currencyUSD", + "label": "Write Cost", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 62, + "panels": [], "title": "Volume Server", - "titleSize": "h6" + "type": "row" }, { - "collapse": true, - "height": 251, - "panels": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 65 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Store Request Duration 99th percentile", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[$__rate_interval])) by (le, exported_instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "B" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(SeaweedFS_filerStore_request_total [1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Store QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[$__rate_interval])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "C" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "thresholds": [], + "timeRegions": [], + "title": "Volume Server Request Duration 99th percentile", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 65 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "total", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(SeaweedFS_volumeServer_request_total[$__rate_interval])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Volume Server QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 72 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(SeaweedFS_volumeServer_volumes) by (collection, type)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collection}} {{type}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(SeaweedFS_volumeServer_volumes)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Volume Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 79 + }, + "hiddenSeries": false, + "id": 50, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (collection, type)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collection}} {{type}}", + "refId": "A" + }, + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Used Disk Space by Collection and Type", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (exported_instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Used Disk Space by Host", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 63, + "panels": [], "title": "Filer Store", - "titleSize": "h6" + "type": "row" }, { - "collapse": true, - "height": 242, - "panels": [ + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 52, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_memstats_alloc_bytes{exported_job=\"filer\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "bytes allocated", - "refId": "B" - }, - { - "expr": "rate(go_memstats_alloc_bytes_total{exported_job=\"filer\"}[30s])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "alloc rate", - "refId": "A" - }, - { - "expr": "go_memstats_stack_inuse_bytes{exported_job=\"filer\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "stack inuse", - "refId": "C" - }, - { - "expr": "go_memstats_heap_inuse_bytes{exported_job=\"filer\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "heap inuse", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Go Memory Stats", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 54, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_gc_duration_seconds{exported_job=\"filer\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{quantile}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Go GC duration quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 53, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{exported_job=\"filer\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{exported_instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Filer Go Routines", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[$__rate_interval])) by (le, type))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "thresholds": [], + "timeRegions": [], + "title": "Filer Store Request Duration 99th percentile", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 94 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(SeaweedFS_filerStore_request_total [$__rate_interval])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filer Store QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 64, + "panels": [], "title": "Filer Instances", - "titleSize": "h6" + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 52, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_alloc_bytes{job=\"seaweedfs-filer\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "bytes allocated", + "refId": "B" + }, + { + "exemplar": true, + "expr": "rate(go_memstats_alloc_bytes_total{job=\"seaweedfs-filer\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "alloc rate", + "refId": "A" + }, + { + "exemplar": true, + "expr": "go_memstats_stack_inuse_bytes{job=\"seaweedfs-filer\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "stack inuse", + "refId": "C" + }, + { + "exemplar": true, + "expr": "go_memstats_heap_inuse_bytes{job=\"seaweedfs-filer\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "heap inuse", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filer Go Memory Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "Bps", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 102 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_gc_duration_seconds{job=\"seaweedfs-filer\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filer Go GC duration quantiles", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "Bps", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 109 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.15", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_goroutines{job=\"seaweedfs-filer\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{exported_instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Filer Go Routines", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "schemaVersion": 14, + "refresh": false, + "schemaVersion": 36, "style": "dark", "tags": [], "templating": { "list": [ { "current": { - "selected": true, + "selected": false, "text": "Prometheus", "value": "Prometheus" }, - "description": null, - "error": null, "hide": 0, "includeAll": false, "label": "Datasource", @@ -1835,7 +2851,7 @@ ] }, "time": { - "from": "now-30d", + "from": "now-1d", "to": "now" }, "timepicker": { @@ -1865,5 +2881,5 @@ }, "timezone": "browser", "title": "SeaweedFS", - "version": 2 + "version": 3 } \ No newline at end of file From e6385e5e8b3b18f962df6c0dc6065c2ffc2452b6 Mon Sep 17 00:00:00 2001 From: Lars Lehtonen Date: Fri, 1 Sep 2023 13:10:45 -0700 Subject: [PATCH 25/47] weed/util: fix dropped errors (#4803) --- weed/util/http_util.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/weed/util/http_util.go b/weed/util/http_util.go index bb1a32ede..ef4b29158 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -54,6 +54,9 @@ func Post(url string, values url.Values) ([]byte, error) { func Get(url string) ([]byte, bool, error) { request, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, true, err + } request.Header.Add("Accept-Encoding", "gzip") response, err := client.Do(request) @@ -66,6 +69,9 @@ func Get(url string) ([]byte, bool, error) { switch response.Header.Get("Content-Encoding") { case "gzip": reader, err = gzip.NewReader(response.Body) + if err != nil { + return nil, true, err + } defer reader.Close() default: reader = response.Body @@ -253,6 +259,9 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC switch contentEncoding { case "gzip": reader, err = gzip.NewReader(r.Body) + if err != nil { + return 0, err + } defer reader.Close() default: reader = r.Body @@ -400,6 +409,9 @@ func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*htt switch contentEncoding { case "gzip": reader, err = gzip.NewReader(r.Body) + if err != nil { + return nil, nil, err + } default: reader = r.Body } From 7374ceecda0b533d9b5a0178615ee6ae58ce23c3 Mon Sep 17 00:00:00 2001 From: fibbanachi <49879947+Programmeris@users.noreply.github.com> Date: Sat, 2 Sep 2023 23:26:27 +0300 Subject: [PATCH 26/47] improve k8s probes templating in helm chart (#4805) --- .../templates/filer-statefulset.yaml | 32 +++--- .../templates/master-statefulset.yaml | 32 +++--- .../seaweedfs/templates/s3-deployment.yaml | 32 +++--- .../templates/volume-statefulset.yaml | 32 +++--- k8s/charts/seaweedfs/values.yaml | 103 ++++++++++++++++++ 5 files changed, 175 insertions(+), 56 deletions(-) diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index 48279dad7..244fd5972 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -224,26 +224,30 @@ spec: name: metrics - containerPort: {{ .Values.filer.grpcPort }} #name: swfs-filer-grpc + {{- if .Values.filer.readinessProbe.enabled }} readinessProbe: httpGet: - path: / + path: {{ .Values.filer.readinessProbe.httpGet.path }} port: {{ .Values.filer.port }} - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 100 - timeoutSeconds: 10 + scheme: {{ .Values.filer.readinessProbe.scheme }} + initialDelaySeconds: {{ .Values.filer.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.filer.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.filer.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.filer.readinessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.filer.readinessProbe.timeoutSeconds }} + {{- end }} + {{- if .Values.filer.livenessProbe.enabled }} livenessProbe: httpGet: - path: / + path: {{ .Values.filer.livenessProbe.httpGet.path }} port: {{ .Values.filer.port }} - scheme: HTTP - initialDelaySeconds: 20 - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 5 - timeoutSeconds: 10 + scheme: {{ .Values.filer.livenessProbe.scheme }} + initialDelaySeconds: {{ .Values.filer.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.filer.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.filer.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.filer.livenessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.filer.livenessProbe.timeoutSeconds }} + {{- end }} {{- if .Values.filer.resources }} resources: {{ tpl .Values.filer.resources . | nindent 12 | trim }} diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index ba4b54c27..0acc9c7b0 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -177,26 +177,30 @@ spec: {{- end }} - containerPort: {{ .Values.master.grpcPort }} #name: swfs-master-grpc + {{- if .Values.master.readinessProbe.enabled }} readinessProbe: httpGet: - path: /cluster/status + path: {{ .Values.master.readinessProbe.httpGet.path }} port: {{ .Values.master.port }} - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 45 - successThreshold: 2 - failureThreshold: 100 - timeoutSeconds: 10 + scheme: {{ .Values.master.readinessProbe.scheme }} + initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.master.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.master.readinessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }} + {{- end }} + {{- if .Values.master.livenessProbe.enabled }} livenessProbe: httpGet: - path: /cluster/status + path: {{ .Values.master.livenessProbe.httpGet.path }} port: {{ .Values.master.port }} - scheme: HTTP - initialDelaySeconds: 20 - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 4 - timeoutSeconds: 10 + scheme: {{ .Values.master.livenessProbe.scheme }} + initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.master.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.master.livenessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }} + {{- end }} {{- if .Values.master.resources }} resources: {{ tpl .Values.master.resources . | nindent 12 | trim }} diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index 2fe3fee92..3cf3008df 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -146,26 +146,30 @@ spec: - containerPort: {{ .Values.s3.metricsPort }} name: "metrics" {{- end }} + {{- if .Values.s3.readinessProbe.enabled }} readinessProbe: httpGet: - path: /status + path: {{ .Values.s3.readinessProbe.httpGet.path }} port: {{ .Values.s3.port }} - scheme: HTTP - initialDelaySeconds: 15 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 100 - timeoutSeconds: 10 + scheme: {{ .Values.s3.readinessProbe.scheme }} + initialDelaySeconds: {{ .Values.s3.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.s3.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.s3.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.s3.readinessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.s3.readinessProbe.timeoutSeconds }} + {{- end }} + {{- if .Values.s3.livenessProbe.enabled }} livenessProbe: httpGet: - path: /status + path: {{ .Values.s3.livenessProbe.httpGet.path }} port: {{ .Values.s3.port }} - scheme: HTTP - initialDelaySeconds: 20 - periodSeconds: 60 - successThreshold: 1 - failureThreshold: 20 - timeoutSeconds: 10 + scheme: {{ .Values.s3.livenessProbe.scheme }} + initialDelaySeconds: {{ .Values.s3.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.s3.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.s3.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.s3.livenessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.s3.livenessProbe.timeoutSeconds }} + {{- end }} {{- if .Values.s3.resources }} resources: {{ tpl .Values.s3.resources . | nindent 12 | trim }} diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 344821a4d..51ba97a0a 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -183,26 +183,30 @@ spec: {{- end }} - containerPort: {{ .Values.volume.grpcPort }} name: swfs-vol-grpc + {{- if .Values.volume.readinessProbe.enabled }} readinessProbe: httpGet: - path: /status + path: {{ .Values.volume.readinessProbe.httpGet.path }} port: {{ .Values.volume.port }} - scheme: HTTP - initialDelaySeconds: 15 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 100 - timeoutSeconds: 30 + scheme: {{ .Values.volume.readinessProbe.scheme }} + initialDelaySeconds: {{ .Values.volume.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.volume.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.volume.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.volume.readinessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.volume.readinessProbe.timeoutSeconds }} + {{- end }} + {{- if .Values.volume.livenessProbe.enabled }} livenessProbe: httpGet: - path: /status + path: {{ .Values.volume.livenessProbe.httpGet.path }} port: {{ .Values.volume.port }} - scheme: HTTP - initialDelaySeconds: 20 - periodSeconds: 90 - successThreshold: 1 - failureThreshold: 4 - timeoutSeconds: 30 + scheme: {{ .Values.volume.livenessProbe.scheme }} + initialDelaySeconds: {{ .Values.volume.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.volume.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.volume.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.volume.livenessProbe.failureThreshold }} + timeoutSeconds: {{ .Values.volume.livenessProbe.timeoutSeconds }} + {{- end }} {{- if .Values.volume.resources }} resources: {{ tpl .Values.volume.resources . | nindent 12 | trim }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 786200bec..8ffb24b18 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -160,6 +160,32 @@ master: WEED_MASTER_VOLUME_GROWTH_COPY_3: 3 WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + # used to configure livenessProbe on master-server containers + # + livenessProbe: + enabled: true + httpGet: + path: /cluster/status + scheme: HTTP + initialDelaySeconds: 20 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 4 + timeoutSeconds: 10 + + # used to configure readinessProbe on master-server containers + # + readinessProbe: + enabled: true + httpGet: + path: /cluster/status + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 45 + successThreshold: 2 + failureThreshold: 100 + timeoutSeconds: 10 + volume: enabled: true repository: null @@ -277,6 +303,32 @@ volume: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ serviceAccountName: "" + # used to configure livenessProbe on volume-server containers + # + livenessProbe: + enabled: true + httpGet: + path: /status + scheme: HTTP + initialDelaySeconds: 20 + periodSeconds: 90 + successThreshold: 1 + failureThreshold: 4 + timeoutSeconds: 30 + + # used to configure readinessProbe on volume-server containers + # + readinessProbe: + enabled: true + httpGet: + path: /status + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 30 + filer: enabled: true repository: null @@ -431,6 +483,31 @@ filer: # directories under this folder will be automatically creating a separate bucket WEED_FILER_BUCKETS_FOLDER: "/buckets" + # used to configure livenessProbe on filer containers + # + livenessProbe: + enabled: true + httpGet: + path: / + scheme: HTTP + initialDelaySeconds: 20 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 5 + timeoutSeconds: 10 + + # used to configure readinessProbe on filer containers + # + readinessProbe: + enabled: true + httpGet: + path: / + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 10 # secret env variables secretExtraEnvironmentVars: [] @@ -518,6 +595,32 @@ s3: storageClass: "" hostPathPrefix: /storage + # used to configure livenessProbe on s3 containers + # + livenessProbe: + enabled: true + httpGet: + path: /status + scheme: HTTP + initialDelaySeconds: 20 + periodSeconds: 60 + successThreshold: 1 + failureThreshold: 20 + timeoutSeconds: 10 + + # used to configure readinessProbe on s3 containers + # + readinessProbe: + enabled: true + httpGet: + path: /status + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 10 + certificates: commonName: "SeaweedFS CA" ipAddresses: [] From 5c613f18ad4dd4a5023b55ba9990dccf31aaf150 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:03:03 -0700 Subject: [PATCH 27/47] Bump github.com/go-redsync/redsync/v4 from 4.8.1 to 4.8.2 (#4807) Bumps [github.com/go-redsync/redsync/v4](https://github.com/go-redsync/redsync) from 4.8.1 to 4.8.2. - [Release notes](https://github.com/go-redsync/redsync/releases) - [Commits](https://github.com/go-redsync/redsync/compare/v4.8.1...v4.8.2) --- updated-dependencies: - dependency-name: github.com/go-redsync/redsync/v4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f0d056ebb..5b03a2de9 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-errors/errors v1.1.1 // indirect github.com/go-redis/redis/v8 v8.11.5 - github.com/go-redsync/redsync/v4 v4.8.1 + github.com/go-redsync/redsync/v4 v4.8.2 github.com/go-sql-driver/mysql v1.7.1 github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gocql/gocql v1.6.0 diff --git a/go.sum b/go.sum index 93675551b..7884db173 100644 --- a/go.sum +++ b/go.sum @@ -217,6 +217,7 @@ github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= @@ -300,6 +301,9 @@ github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -309,8 +313,8 @@ github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRf github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-redsync/redsync/v4 v4.8.1 h1:rq2RvdTI0obznMdxKUWGdmmulo7lS9yCzb8fgDKOlbM= -github.com/go-redsync/redsync/v4 v4.8.1/go.mod h1:LmUAsQuQxhzZAoGY7JS6+dNhNmZyonMZiiEDY9plotM= +github.com/go-redsync/redsync/v4 v4.8.2 h1:/b+jidlwOnSOl605hmNsd+ohfr/qB1EpLiKnCYkiw28= +github.com/go-redsync/redsync/v4 v4.8.2/go.mod h1:83QPGYHk0Wt2LhFo3n9xCrUGPQrQvejVTVR08KX032g= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -370,8 +374,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= -github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= +github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -415,6 +419,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= @@ -556,6 +561,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk= github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= @@ -631,6 +637,7 @@ github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItc github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -639,11 +646,28 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.8.0/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= @@ -656,6 +680,7 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU= @@ -745,6 +770,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rueian/rueidis v0.0.93 h1:cG905akj2+QyHx0x9y4mN0K8vLi6M94QiyoLulXS3l0= +github.com/rueian/rueidis v0.0.93/go.mod h1:lo6LBci0D986usi5Wxjb4RVNaWENKYbHZSnufGJ9bTE= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30= @@ -881,6 +908,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8= github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4= @@ -914,6 +942,11 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.12.0/go.mod h1:geaoz0L0r1BEOR81k7/n9W4TCXYCJ7bPO7K374jQHG0= +go.opentelemetry.io/otel/metric v0.35.0/go.mod h1:qAcbhaTRFU6uG8QM7dDo7XvFsWcugziq/5YI065TokQ= +go.opentelemetry.io/otel/sdk v1.12.0/go.mod h1:WYcvtgquYvgODEvxOry5owO2y9MyciW7JqMz6cpXShE= +go.opentelemetry.io/otel/sdk/metric v0.35.0/go.mod h1:eDyp1GxSiwV98kr7w4pzrszQh/eze9MqBqPd2bCPmyE= +go.opentelemetry.io/otel/trace v1.12.0/go.mod h1:pHlgBynn6s25qJ2szD+Bv+iwKJttjHSI3lUAyf0GNuQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -923,6 +956,7 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -999,7 +1033,10 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -1046,9 +1083,15 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= @@ -1141,8 +1184,12 @@ golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1150,8 +1197,11 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1160,6 +1210,9 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= @@ -1175,6 +1228,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -1241,7 +1296,11 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= From 41ebc9c3cb673a21541c310b9abc531880c3b322 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:03:33 -0700 Subject: [PATCH 28/47] Bump github.com/puzpuzpuz/xsync/v2 from 2.4.1 to 2.5.0 (#4810) Bumps [github.com/puzpuzpuz/xsync/v2](https://github.com/puzpuzpuz/xsync) from 2.4.1 to 2.5.0. - [Release notes](https://github.com/puzpuzpuz/xsync/releases) - [Commits](https://github.com/puzpuzpuz/xsync/compare/v2.4.1...v2.5.0) --- updated-dependencies: - dependency-name: github.com/puzpuzpuz/xsync/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5b03a2de9..a7fa98f36 100644 --- a/go.mod +++ b/go.mod @@ -154,7 +154,7 @@ require ( github.com/hashicorp/raft v1.5.0 github.com/hashicorp/raft-boltdb/v2 v2.2.2 github.com/orcaman/concurrent-map/v2 v2.0.1 - github.com/puzpuzpuz/xsync/v2 v2.4.1 + github.com/puzpuzpuz/xsync/v2 v2.5.0 github.com/rabbitmq/amqp091-go v1.8.1 github.com/rclone/rclone v1.63.1 github.com/schollz/progressbar/v3 v3.13.1 diff --git a/go.sum b/go.sum index 7884db173..fd525ee39 100644 --- a/go.sum +++ b/go.sum @@ -746,8 +746,8 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU= -github.com/puzpuzpuz/xsync/v2 v2.4.1 h1:aGdE1C/HaR/QC6YAFdtZXi60Df8/qBIrs8PKrzkItcM= -github.com/puzpuzpuz/xsync/v2 v2.4.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= +github.com/puzpuzpuz/xsync/v2 v2.5.0 h1:2k4qrO/orvmEXZ3hmtHqIy9XaQtPTwzMZk1+iErpE8c= +github.com/puzpuzpuz/xsync/v2 v2.5.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA= github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d h1:ZyH6ZfA/PzxF4qQS2MgFLXRdw/pWOSNJA7Lq0pkX49Y= From f3dfeb373514ea5684b81d5290b86982ab9a533b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:03:41 -0700 Subject: [PATCH 29/47] Bump github.com/tidwall/gjson from 1.14.4 to 1.16.0 (#4811) Bumps [github.com/tidwall/gjson](https://github.com/tidwall/gjson) from 1.14.4 to 1.16.0. - [Commits](https://github.com/tidwall/gjson/compare/v1.14.4...v1.16.0) --- updated-dependencies: - dependency-name: github.com/tidwall/gjson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a7fa98f36..e2e47535c 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 - github.com/tidwall/gjson v1.14.4 + github.com/tidwall/gjson v1.16.0 github.com/tidwall/match v1.1.1 github.com/tidwall/pretty v1.2.0 // indirect github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 diff --git a/go.sum b/go.sum index fd525ee39..7fbca2501 100644 --- a/go.sum +++ b/go.sum @@ -842,8 +842,8 @@ github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4U github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= +github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= From f07876cb23bfbf435d91506d60bb5e7a50e5888e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:03:49 -0700 Subject: [PATCH 30/47] Bump actions/checkout from 3 to 4 (#4812) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...3df4ab11eba7bda6032a0b82a6bb43b11571feac) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/binaries_dev.yml | 4 ++-- .github/workflows/binaries_release0.yml | 2 +- .github/workflows/binaries_release1.yml | 2 +- .github/workflows/binaries_release2.yml | 2 +- .github/workflows/binaries_release3.yml | 2 +- .github/workflows/binaries_release4.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/container_dev.yml | 2 +- .github/workflows/container_latest.yml | 2 +- .github/workflows/container_release1.yml | 2 +- .github/workflows/container_release2.yml | 2 +- .github/workflows/container_release3.yml | 2 +- .github/workflows/container_release4.yml | 2 +- .github/workflows/container_release5.yml | 2 +- .github/workflows/depsreview.yml | 2 +- .github/workflows/e2e.yml | 2 +- .github/workflows/go.yml | 2 +- .github/workflows/helm_chart_release.yml | 2 +- .github/workflows/helm_ci.yml | 2 +- 19 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/binaries_dev.yml b/.github/workflows/binaries_dev.yml index 2c74d09ed..6e007a797 100644 --- a/.github/workflows/binaries_dev.yml +++ b/.github/workflows/binaries_dev.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Set BUILD_TIME env run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} @@ -87,7 +87,7 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Set BUILD_TIME env run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} diff --git a/.github/workflows/binaries_release0.yml b/.github/workflows/binaries_release0.yml index 245b247f4..38c233083 100644 --- a/.github/workflows/binaries_release0.yml +++ b/.github/workflows/binaries_release0.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22 with: diff --git a/.github/workflows/binaries_release1.yml b/.github/workflows/binaries_release1.yml index a2224e199..88e59ad16 100644 --- a/.github/workflows/binaries_release1.yml +++ b/.github/workflows/binaries_release1.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22 with: diff --git a/.github/workflows/binaries_release2.yml b/.github/workflows/binaries_release2.yml index 5403b7e9c..4da7b5c6e 100644 --- a/.github/workflows/binaries_release2.yml +++ b/.github/workflows/binaries_release2.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22 with: diff --git a/.github/workflows/binaries_release3.yml b/.github/workflows/binaries_release3.yml index a1def6e90..6c660c0ce 100644 --- a/.github/workflows/binaries_release3.yml +++ b/.github/workflows/binaries_release3.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22 with: diff --git a/.github/workflows/binaries_release4.yml b/.github/workflows/binaries_release4.yml index 37f9aa855..1fce19261 100644 --- a/.github/workflows/binaries_release4.yml +++ b/.github/workflows/binaries_release4.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22 with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4dbbe1c2f..a44b1ac19 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index d0591455a..d9f99f0b8 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index b7c8b650b..0a34770dc 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index b88871d30..ade6ac75e 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index 2a9f68404..c5f3322fc 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index b4ba92bf6..e22489558 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 0526eb842..6a4dd2656 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index 17d36862f..e7b9db03a 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index 9982ceb98..e73f4a92d 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -9,6 +9,6 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac - name: 'Dependency Review' uses: actions/dependency-review-action@f6fff72a3217f580d5afd49a46826795305b63c7 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 014c43d73..a6a05e756 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -30,7 +30,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Install dependencies run: | diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7cb9283c5..d5b11b93b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -27,7 +27,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2 - name: Get dependencies run: | diff --git a/.github/workflows/helm_chart_release.yml b/.github/workflows/helm_chart_release.yml index 9a6cf2509..6dcf3b209 100644 --- a/.github/workflows/helm_chart_release.yml +++ b/.github/workflows/helm_chart_release.yml @@ -12,7 +12,7 @@ jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac - name: Publish Helm charts uses: stefanprodan/helm-gh-pages@master with: diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index 6129faa53..b5d72ac50 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac with: fetch-depth: 0 From cdd817edf97fca305bb490b139d6a9b185d58f66 Mon Sep 17 00:00:00 2001 From: Patrick Schmidt Date: Fri, 20 Jan 2023 13:12:30 +0100 Subject: [PATCH 31/47] Improve S3 request signing performance This change is caching HMAC hashers for repeated use in subsequent requests and chunks, so they don't have to be initialized from scratch every time. On my local computer this gives me ~5-6 times faster signature calculation and ~5-6.5% more throughput in S3 requests. The smaller the payload the better the throughput gets. --- weed/s3api/auth_credentials.go | 6 ++- weed/s3api/auth_signature_v4.go | 78 ++++++++++++++++++++-------- weed/s3api/auto_signature_v4_test.go | 25 ++++++--- weed/s3api/chunked_reader_v4.go | 60 +++++++++++---------- 4 files changed, 112 insertions(+), 57 deletions(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 876acd7cf..f2d057b90 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -2,12 +2,13 @@ package s3api import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "net/http" "os" "strings" "sync" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -31,6 +32,8 @@ type IdentityAccessManagement struct { identities []*Identity isAuthEnabled bool domain string + hashes map[string]*sync.Pool + hashMu sync.RWMutex } type Identity struct { @@ -77,6 +80,7 @@ func (action Action) getPermission() Permission { func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement { iam := &IdentityAccessManagement{ domain: option.DomainName, + hashes: make(map[string]*sync.Pool), } if option.Config != "" { if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 02a6bd4e0..06cdf67e4 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -23,6 +23,7 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" + "hash" "io" "net/http" "net/url" @@ -30,6 +31,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "unicode/utf8" @@ -151,14 +153,14 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r // Get string to sign from canonical request. stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope()) - // Get hmac signing key. - signingKey := getSigningKey(cred.SecretKey, + // Calculate signature. + newSignature := iam.getSignature( + cred.SecretKey, signV4Values.Credential.scope.date, signV4Values.Credential.scope.region, - signV4Values.Credential.scope.service) - - // Calculate signature. - newSignature := getSignature(signingKey, stringToSign) + signV4Values.Credential.scope.service, + stringToSign, + ) // Verify if signature match. if !compareSignatureV4(newSignature, signV4Values.Signature) { @@ -325,11 +327,14 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http. return s3err.ErrInvalidAccessKeyID } - // Get signing key. - signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, credHeader.scope.service) - // Get signature. - newSignature := getSignature(signingKey, formValues.Get("Policy")) + newSignature := iam.getSignature( + cred.SecretKey, + credHeader.scope.date, + credHeader.scope.region, + credHeader.scope.service, + formValues.Get("Policy"), + ) // Verify signature. if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) { @@ -442,14 +447,14 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s // Get string to sign from canonical request. presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope()) - // Get hmac presigned signing key. - presignedSigningKey := getSigningKey(cred.SecretKey, + // Get new signature. + newSignature := iam.getSignature( + cred.SecretKey, pSignValues.Credential.scope.date, pSignValues.Credential.scope.region, - pSignValues.Credential.scope.service) - - // Get new signature. - newSignature := getSignature(presignedSigningKey, presignedStringToSign) + pSignValues.Credential.scope.service, + presignedStringToSign, + ) // Verify signature. if !compareSignatureV4(req.URL.Query().Get("X-Amz-Signature"), newSignature) { @@ -458,6 +463,38 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s return identity, s3err.ErrNone } +// getSignature +func (iam *IdentityAccessManagement) getSignature(secretKey string, t time.Time, region string, service string, stringToSign string) string { + date := t.Format(yyyymmdd) + hashID := "AWS4" + secretKey + "/" + date + "/" + region + "/" + service + "/" + "aws4_request" + + iam.hashMu.RLock() + pool, ok := iam.hashes[hashID] + iam.hashMu.RUnlock() + + if !ok { + iam.hashMu.Lock() + if pool, ok = iam.hashes[hashID]; !ok { + pool = &sync.Pool{ + New: func() any { + signingKey := getSigningKey(secretKey, date, region, service) + return hmac.New(sha256.New, signingKey) + }, + } + iam.hashes[hashID] = pool + } + iam.hashMu.Unlock() + } + + h := pool.Get().(hash.Hash) + h.Reset() + h.Write([]byte(stringToSign)) + sig := hex.EncodeToString(h.Sum(nil)) + pool.Put(h) + + return sig +} + func contains(list []string, elem string) bool { for _, t := range list { if t == elem { @@ -674,19 +711,14 @@ func sumHMAC(key []byte, data []byte) []byte { } // getSigningKey hmac seed to calculate final signature. -func getSigningKey(secretKey string, t time.Time, region string, service string) []byte { - date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd))) +func getSigningKey(secretKey string, time string, region string, service string) []byte { + date := sumHMAC([]byte("AWS4"+secretKey), []byte(time)) regionBytes := sumHMAC(date, []byte(region)) serviceBytes := sumHMAC(regionBytes, []byte(service)) signingKey := sumHMAC(serviceBytes, []byte("aws4_request")) return signingKey } -// getSignature final signature in hexadecimal form. -func getSignature(signingKey []byte, stringToSign string) string { - return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) -} - // getCanonicalHeaders generate a list of request headers with their values func getCanonicalHeaders(signedHeaders http.Header) string { var headers []string diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index db8bfd8ef..15ca90b93 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -14,6 +14,7 @@ import ( "sort" "strconv" "strings" + "sync" "testing" "time" "unicode/utf8" @@ -114,7 +115,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) { }{ {Request: mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrAccessDenied}, {Request: mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, - {Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, + {Request: mustNewPresignedRequest(iam, "GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, } for i, testCase := range testCases { if _, s3Error := iam.reqSignatureV4Verify(testCase.Request); s3Error != testCase.ErrCode { @@ -123,6 +124,19 @@ func TestCheckAdminRequestAuthType(t *testing.T) { } } +func BenchmarkGetSignature(b *testing.B) { + t := time.Now() + iam := IdentityAccessManagement{ + hashes: make(map[string]*sync.Pool), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + iam.getSignature("secret-key", t, "us-east-1", "s3", "random data") + } +} + // Provides a fully populated http request instance, fails otherwise. func mustNewRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { req, err := newTestRequest(method, urlStr, contentLength, body) @@ -145,10 +159,10 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod // This is similar to mustNewRequest but additionally the request // is presigned with AWS Signature V4, fails if not able to do so. -func mustNewPresignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { +func mustNewPresignedRequest(iam *IdentityAccessManagement, method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { req := mustNewRequest(method, urlStr, contentLength, body, t) cred := &Credential{"access_key_1", "secret_key_1"} - if err := preSignV4(req, cred.AccessKey, cred.SecretKey, int64(10*time.Minute.Seconds())); err != nil { + if err := preSignV4(iam, req, cred.AccessKey, cred.SecretKey, int64(10*time.Minute.Seconds())); err != nil { t.Fatalf("Unable to initialized new signed http request %s", err) } return req @@ -343,7 +357,7 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error { // preSignV4 presign the request, in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. -func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires int64) error { +func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKeyID, secretAccessKey string, expires int64) error { // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return errors.New("Presign cannot be generated without access and secret keys") @@ -370,8 +384,7 @@ func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires i queryStr := strings.Replace(query.Encode(), "+", "%20", -1) canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method) stringToSign := getStringToSign(canonicalRequest, date, scope) - signingKey := getSigningKey(secretAccessKey, date, region, "s3") - signature := getSignature(signingKey, stringToSign) + signature := iam.getSignature(secretAccessKey, date, region, "s3", stringToSign) req.URL.RawQuery = query.Encode() diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index 8ba1bc479..4bf74d025 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -24,36 +24,17 @@ import ( "crypto/sha256" "encoding/hex" "errors" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "hash" "io" "net/http" "time" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/dustin/go-humanize" ) -// getChunkSignature - get chunk signature. -func getChunkSignature(secretKey string, seedSignature string, region string, date time.Time, hashedChunk string) string { - - // Calculate string to sign. - stringToSign := signV4ChunkedAlgorithm + "\n" + - date.Format(iso8601Format) + "\n" + - getScope(date, region) + "\n" + - seedSignature + "\n" + - emptySHA256 + "\n" + - hashedChunk - - // Get hmac signing key. - signingKey := getSigningKey(secretKey, date, region, "s3") - - // Calculate signature. - newSignature := getSignature(signingKey, stringToSign) - - return newSignature -} - // calculateSeedSignature - Calculate seed signature in accordance with // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html // @@ -124,11 +105,14 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr // Get string to sign from canonical request. stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope()) - // Get hmac signing key. - signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3") - // Calculate signature. - newSignature := getSignature(signingKey, stringToSign) + newSignature := iam.getSignature( + cred.SecretKey, + signV4Values.Credential.scope.date, + region, + "s3", + stringToSign, + ) // Verify if signature match. if !compareSignatureV4(newSignature, signV4Values.Signature) { @@ -163,6 +147,7 @@ func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) ( region: region, chunkSHA256Writer: sha256.New(), state: readChunkHeader, + iam: iam, }, s3err.ErrNone } @@ -180,6 +165,7 @@ type s3ChunkedReader struct { chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data. n uint64 // Unread bytes in chunk err error + iam *IdentityAccessManagement } // Read chunk reads the chunk token signature portion. @@ -296,7 +282,7 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { // Calculate the hashed chunk. hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)) // Calculate the chunk signature. - newSignature := getChunkSignature(cr.cred.SecretKey, cr.seedSignature, cr.region, cr.seedDate, hashedChunk) + newSignature := cr.getChunkSignature(hashedChunk) if !compareSignatureV4(cr.chunkSignature, newSignature) { // Chunk signature doesn't match we return signature does not match. cr.err = errors.New("chunk signature does not match") @@ -317,6 +303,26 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { } } +// getChunkSignature - get chunk signature. +func (cr *s3ChunkedReader) getChunkSignature(hashedChunk string) string { + // Calculate string to sign. + stringToSign := signV4ChunkedAlgorithm + "\n" + + cr.seedDate.Format(iso8601Format) + "\n" + + getScope(cr.seedDate, cr.region) + "\n" + + cr.seedSignature + "\n" + + emptySHA256 + "\n" + + hashedChunk + + // Calculate signature. + return cr.iam.getSignature( + cr.cred.SecretKey, + cr.seedDate, + cr.region, + "s3", + stringToSign, + ) +} + // readCRLF - check if reader only has '\r\n' CRLF character. // returns malformed encoding if it doesn't. func readCRLF(reader io.Reader) error { From 98dcec0ee2023d24913941bfef7ecc5a7d2b78f8 Mon Sep 17 00:00:00 2001 From: Patrick Schmidt Date: Fri, 1 Sep 2023 19:57:04 +0200 Subject: [PATCH 32/47] Clean up old signature hash pools --- weed/s3api/auth_credentials.go | 6 ++- weed/s3api/auth_signature_v4.go | 66 +++++++++++++++++++++------- weed/s3api/auto_signature_v4_test.go | 3 +- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index f2d057b90..234dc100b 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -33,6 +33,7 @@ type IdentityAccessManagement struct { isAuthEnabled bool domain string hashes map[string]*sync.Pool + hashCounters map[string]*int32 hashMu sync.RWMutex } @@ -79,8 +80,9 @@ func (action Action) getPermission() Permission { func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement { iam := &IdentityAccessManagement{ - domain: option.DomainName, - hashes: make(map[string]*sync.Pool), + domain: option.DomainName, + hashes: make(map[string]*sync.Pool), + hashCounters: make(map[string]*int32), } if option.Config != "" { if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 06cdf67e4..04548cc6f 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -32,6 +32,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "unicode/utf8" @@ -463,36 +464,67 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s return identity, s3err.ErrNone } -// getSignature func (iam *IdentityAccessManagement) getSignature(secretKey string, t time.Time, region string, service string, stringToSign string) string { + pool := iam.getSignatureHashPool(secretKey, t, region, service) + h := pool.Get().(hash.Hash) + defer pool.Put(h) + + h.Reset() + h.Write([]byte(stringToSign)) + sig := hex.EncodeToString(h.Sum(nil)) + + return sig +} + +func (iam *IdentityAccessManagement) getSignatureHashPool(secretKey string, t time.Time, region string, service string) *sync.Pool { + // Build a caching key for the pool. date := t.Format(yyyymmdd) hashID := "AWS4" + secretKey + "/" + date + "/" + region + "/" + service + "/" + "aws4_request" + // Try to find an existing pool and return it. iam.hashMu.RLock() pool, ok := iam.hashes[hashID] iam.hashMu.RUnlock() if !ok { iam.hashMu.Lock() - if pool, ok = iam.hashes[hashID]; !ok { - pool = &sync.Pool{ - New: func() any { - signingKey := getSigningKey(secretKey, date, region, service) - return hmac.New(sha256.New, signingKey) - }, - } - iam.hashes[hashID] = pool - } - iam.hashMu.Unlock() + defer iam.hashMu.Unlock() + pool, ok = iam.hashes[hashID] } - h := pool.Get().(hash.Hash) - h.Reset() - h.Write([]byte(stringToSign)) - sig := hex.EncodeToString(h.Sum(nil)) - pool.Put(h) + if ok { + atomic.StoreInt32(iam.hashCounters[hashID], 1) + return pool + } - return sig + // Create a pool that returns HMAC hashers for the requested parameters to avoid expensive re-initializing + // of new instances on every request. + iam.hashes[hashID] = &sync.Pool{ + New: func() any { + signingKey := getSigningKey(secretKey, date, region, service) + return hmac.New(sha256.New, signingKey) + }, + } + iam.hashCounters[hashID] = new(int32) + + // Clean up unused pools automatically after one hour of inactivity + ticker := time.NewTicker(time.Hour) + go func() { + for range ticker.C { + old := atomic.SwapInt32(iam.hashCounters[hashID], 0) + if old == 0 { + break + } + } + + ticker.Stop() + iam.hashMu.Lock() + delete(iam.hashes, hashID) + delete(iam.hashCounters, hashID) + iam.hashMu.Unlock() + }() + + return iam.hashes[hashID] } func contains(list []string, elem string) bool { diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index 15ca90b93..8d0b677f8 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -127,7 +127,8 @@ func TestCheckAdminRequestAuthType(t *testing.T) { func BenchmarkGetSignature(b *testing.B) { t := time.Now() iam := IdentityAccessManagement{ - hashes: make(map[string]*sync.Pool), + hashes: make(map[string]*sync.Pool), + hashCounters: make(map[string]*int32), } b.ReportAllocs() From f555bb0ca4984f0b7228806137f5ba615af0acd0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 15:18:17 +0000 Subject: [PATCH 33/47] Bump github.com/fclairamb/ftpserverlib from 0.21.0 to 0.22.0 Bumps [github.com/fclairamb/ftpserverlib](https://github.com/fclairamb/ftpserverlib) from 0.21.0 to 0.22.0. - [Release notes](https://github.com/fclairamb/ftpserverlib/releases) - [Commits](https://github.com/fclairamb/ftpserverlib/compare/v0.21.0...v0.22.0) --- updated-dependencies: - dependency-name: github.com/fclairamb/ftpserverlib dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e2e47535c..d114f9c79 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect - github.com/fclairamb/ftpserverlib v0.21.0 + github.com/fclairamb/ftpserverlib v0.22.0 github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-errors/errors v1.1.1 // indirect github.com/go-redis/redis/v8 v8.11.5 diff --git a/go.sum b/go.sum index 7fbca2501..c06ad21d5 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= -github.com/fclairamb/ftpserverlib v0.21.0 h1:QO4ex827FU6Y7FNi1cj4dmAs6bcmy+UtWcX5yzVzFAw= -github.com/fclairamb/ftpserverlib v0.21.0/go.mod h1:03sR5yGPYyUH/8hFKML02SVNLY7A//3qIy0q0ZJGhTw= +github.com/fclairamb/ftpserverlib v0.22.0 h1:PqzyD6YxS5sdb4fAdXUFSODTo8DelsVAOh3LgeR4VXs= +github.com/fclairamb/ftpserverlib v0.22.0/go.mod h1:dI9/yw/KfJ0g4wmRK8ZukUfqakLr6ZTf9VDydKoLy90= github.com/fclairamb/go-log v0.4.1 h1:rLtdSG9x2pK41AIAnE8WYpl05xBJfw1ZyYxZaXFcBsM= github.com/fclairamb/go-log v0.4.1/go.mod h1:sw1KvnkZ4wKCYkvy4SL3qVZcJSWFP8Ure4pM3z+KNn4= github.com/fluent/fluent-logger-golang v1.9.0 h1:zUdY44CHX2oIUc7VTNZc+4m+ORuO/mldQDA7czhWXEg= From c3810520fcb208ce6a1efbc1e69dc5f33861d262 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:12:30 +0000 Subject: [PATCH 34/47] Bump golang.org/x/sys from 0.11.0 to 0.12.0 Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.11.0 to 0.12.0. - [Commits](https://github.com/golang/sys/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d114f9c79..658c4a2d3 100644 --- a/go.mod +++ b/go.mod @@ -117,7 +117,7 @@ require ( golang.org/x/image v0.11.0 golang.org/x/net v0.14.0 golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sys v0.11.0 + golang.org/x/sys v0.12.0 golang.org/x/text v0.12.0 // indirect golang.org/x/tools v0.11.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index c06ad21d5..65192201e 100644 --- a/go.sum +++ b/go.sum @@ -1205,8 +1205,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From a879e1bbb58808840ec27e3c411fe1ffface2485 Mon Sep 17 00:00:00 2001 From: Lars Lehtonen Date: Wed, 6 Sep 2023 07:19:04 -0700 Subject: [PATCH 35/47] weed/storage/erasure_coding: remove unused err from encodeDatFile() signature --- weed/storage/erasure_coding/ec_encoder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go index 7d68de2e6..a46643f57 100644 --- a/weed/storage/erasure_coding/ec_encoder.go +++ b/weed/storage/erasure_coding/ec_encoder.go @@ -79,7 +79,7 @@ func generateEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, } glog.V(0).Infof("encodeDatFile %s.dat size:%d", baseFileName, fi.Size()) - err = encodeDatFile(fi.Size(), err, baseFileName, bufferSize, largeBlockSize, file, smallBlockSize) + err = encodeDatFile(fi.Size(), baseFileName, bufferSize, largeBlockSize, file, smallBlockSize) if err != nil { return fmt.Errorf("encodeDatFile: %v", err) } @@ -195,7 +195,7 @@ func encodeDataOneBatch(file *os.File, enc reedsolomon.Encoder, startOffset, blo return nil } -func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSize int, largeBlockSize int64, file *os.File, smallBlockSize int64) error { +func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, largeBlockSize int64, file *os.File, smallBlockSize int64) error { var processedSize int64 From d762b12e112233f8840031a70c2daefd4df1b8b6 Mon Sep 17 00:00:00 2001 From: Lars Lehtonen Date: Wed, 6 Sep 2023 07:20:24 -0700 Subject: [PATCH 36/47] weed/storage/erasure_coding: fix dropped test error --- weed/storage/erasure_coding/ec_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go index e66a83200..676ebdf2c 100644 --- a/weed/storage/erasure_coding/ec_test.go +++ b/weed/storage/erasure_coding/ec_test.go @@ -60,6 +60,9 @@ func validateFiles(baseFileName string) error { } ecFiles, err := openEcFiles(baseFileName, true) + if err != nil { + return fmt.Errorf("error opening ec files: %w", err) + } defer closeEcFiles(ecFiles) err = nm.AscendingVisit(func(value needle_map.NeedleValue) error { From f24c7e803f0a14d35e2bb40a2ed838c80b8cf2e8 Mon Sep 17 00:00:00 2001 From: Lars Lehtonen Date: Wed, 6 Sep 2023 07:21:40 -0700 Subject: [PATCH 37/47] weed/storage/erasure_coding: Close() after error handling --- weed/storage/erasure_coding/ec_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go index 676ebdf2c..aa46cf6d4 100644 --- a/weed/storage/erasure_coding/ec_test.go +++ b/weed/storage/erasure_coding/ec_test.go @@ -43,10 +43,10 @@ func TestEncodingDecoding(t *testing.T) { func validateFiles(baseFileName string) error { nm, err := readNeedleMap(baseFileName) - defer nm.Close() if err != nil { return fmt.Errorf("readNeedleMap: %v", err) } + defer nm.Close() datFile, err := os.OpenFile(baseFileName+".dat", os.O_RDONLY, 0) if err != nil { From 07c2d75ac7cc2b8652203d9d4d59461903d20d4d Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 10 Sep 2023 22:03:44 -0700 Subject: [PATCH 38/47] ensure single threaded access --- weed/operation/assign_file_id.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index a4753d234..f6d0e0110 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -8,6 +8,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "google.golang.org/grpc" + "sync" ) type VolumeAssignRequest struct { @@ -66,9 +67,13 @@ func (ap *AssignProxy) Assign(primaryRequest *VolumeAssignRequest, alternativeRe type singleThreadAssignProxy struct { assignClient master_pb.Seaweed_StreamAssignClient + sync.Mutex } func (ap *singleThreadAssignProxy) doAssign(grpcConnection *grpc.ClientConn, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (ret *AssignResult, err error) { + ap.Lock() + defer ap.Unlock() + if ap.assignClient == nil { client := master_pb.NewSeaweedClient(grpcConnection) ap.assignClient, err = client.StreamAssign(context.Background()) From 5407b8676669a8e6389121b37800301b5e5d4dc2 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 10 Sep 2023 22:05:41 -0700 Subject: [PATCH 39/47] 3.56 --- k8s/charts/seaweedfs/Chart.yaml | 4 ++-- weed/util/constants.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index ec0fefb55..ef588f55d 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "3.55" -version: "3.55" +appVersion: "3.56" +version: "3.56" diff --git a/weed/util/constants.go b/weed/util/constants.go index 7123cc3b7..f6f282bdc 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION_NUMBER = fmt.Sprintf("%.02f", 3.55) + VERSION_NUMBER = fmt.Sprintf("%.02f", 3.56) VERSION = sizeLimit + " " + VERSION_NUMBER COMMIT = "" ) From e3a7c503c3bbfe9bec08af7b64a7558f9b6b0d23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:37:05 +0000 Subject: [PATCH 40/47] Bump actions/dependency-review-action from 3.0.8 to 3.1.0 Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 3.0.8 to 3.1.0. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/f6fff72a3217f580d5afd49a46826795305b63c7...6c5ccdad469c9f8a2996bfecaec55a631a347034) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/depsreview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index e73f4a92d..ca4f2e720 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -11,4 +11,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac - name: 'Dependency Review' - uses: actions/dependency-review-action@f6fff72a3217f580d5afd49a46826795305b63c7 + uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 From c097994f6393bc50a529b8e37e0933205f729034 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:37:02 +0000 Subject: [PATCH 41/47] Bump docker/build-push-action from 4.1.1 to 4.2.1 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4.1.1 to 4.2.1. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/2eb1c1961a95fc15694676618e422e8ba1d63825...0a97817b6ade9f46837855d676c4cca3a2471fc9) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/container_dev.yml | 2 +- .github/workflows/container_latest.yml | 2 +- .github/workflows/container_release1.yml | 2 +- .github/workflows/container_release2.yml | 2 +- .github/workflows/container_release3.yml | 2 +- .github/workflows/container_release4.yml | 2 +- .github/workflows/container_release5.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index d9f99f0b8..23d6bbc1b 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -56,7 +56,7 @@ jobs: password: ${{ secrets.GHCR_TOKEN }} - name: Build - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 0a34770dc..99f18941f 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -57,7 +57,7 @@ jobs: password: ${{ secrets.GHCR_TOKEN }} - name: Build - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index ade6ac75e..1046880ee 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -47,7 +47,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index c5f3322fc..385c72f2e 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -48,7 +48,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index e22489558..e8713ffcb 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -48,7 +48,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 6a4dd2656..199652ad3 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -47,7 +47,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index e7b9db03a..31c9384a0 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -47,7 +47,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} From 324f3fbb153ab95d7ddf1618db00c360476286b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:12:50 +0000 Subject: [PATCH 42/47] Bump modernc.org/mathutil from 1.5.0 to 1.6.0 Bumps [modernc.org/mathutil](https://gitlab.com/cznic/mathutil) from 1.5.0 to 1.6.0. - [Commits](https://gitlab.com/cznic/mathutil/compare/v1.5.0...v1.6.0) --- updated-dependencies: - dependency-name: modernc.org/mathutil dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 658c4a2d3..5d61bc62c 100644 --- a/go.mod +++ b/go.mod @@ -131,7 +131,7 @@ require ( modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect modernc.org/libc v1.24.1 // indirect - modernc.org/mathutil v1.5.0 + modernc.org/mathutil v1.6.0 modernc.org/memory v1.6.0 // indirect modernc.org/opt v0.1.3 // indirect modernc.org/sqlite v1.25.0 diff --git a/go.sum b/go.sum index 65192201e..45dbf8296 100644 --- a/go.sum +++ b/go.sum @@ -1484,8 +1484,8 @@ modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= modernc.org/memory v1.6.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= From d99d99a911b99f29bf783da38681c7245af8d791 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:11:48 +0000 Subject: [PATCH 43/47] Bump github.com/google/uuid from 1.3.0 to 1.3.1 Bumps [github.com/google/uuid](https://github.com/google/uuid) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/google/uuid/releases) - [Changelog](https://github.com/google/uuid/blob/master/CHANGELOG.md) - [Commits](https://github.com/google/uuid/compare/v1.3.0...v1.3.1) --- updated-dependencies: - dependency-name: github.com/google/uuid dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5d61bc62c..235d4833b 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 github.com/google/go-cmp v0.5.9 // indirect - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/google/wire v0.5.0 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/mux v1.8.0 diff --git a/go.sum b/go.sum index 45dbf8296..f7e7d8847 100644 --- a/go.sum +++ b/go.sum @@ -427,8 +427,9 @@ github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= From b28595d8db181c40f6d24743063c9d20e7c4a9e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:10:59 +0000 Subject: [PATCH 44/47] Bump google.golang.org/grpc from 1.57.0 to 1.58.0 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.57.0 to 1.58.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.57.0...v1.58.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 235d4833b..af48346a0 100644 --- a/go.mod +++ b/go.mod @@ -124,7 +124,7 @@ require ( google.golang.org/api v0.134.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect - google.golang.org/grpc v1.57.0 + google.golang.org/grpc v1.58.0 google.golang.org/protobuf v1.31.0 gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/b v1.0.0 // indirect diff --git a/go.sum b/go.sum index f7e7d8847..f8d047c69 100644 --- a/go.sum +++ b/go.sum @@ -1412,8 +1412,8 @@ google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= +google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM= google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 h1:0emxaJWaG6CfrA9Nbe4aHWbFz5AXw2QPEJP0/f42LCE= From 962461fc33aeaabdbe550e24f73ef5a960be9691 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:11:42 +0000 Subject: [PATCH 45/47] Bump github.com/aws/aws-sdk-go-v2/credentials from 1.13.35 to 1.13.37 Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.13.35 to 1.13.37. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.13.35...credentials/v1.13.37) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index af48346a0..6483322c6 100644 --- a/go.mod +++ b/go.mod @@ -145,7 +145,7 @@ require ( github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.33 - github.com/aws/aws-sdk-go-v2/credentials v1.13.35 + github.com/aws/aws-sdk-go-v2/credentials v1.13.37 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 github.com/fluent/fluent-logger-golang v1.9.0 github.com/golang-jwt/jwt/v5 v5.0.0 @@ -191,8 +191,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect github.com/aws/smithy-go v1.14.2 // indirect github.com/benbjohnson/clock v1.3.0 // indirect diff --git a/go.sum b/go.sum index f8d047c69..e36e2e0f0 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,8 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12/go.mod h1:TDCkEAkM github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= -github.com/aws/aws-sdk-go-v2/credentials v1.13.35 h1:QpsNitYJu0GgvMBLUIYu9H4yryA5kMksjeIVQfgXrt8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.35/go.mod h1:o7rCaLtvK0hUggAGclf76mNGGkaG5a9KWlp+d9IpcV8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.37 h1:BvEdm09+ZEh2XtN+PVHPcYwKY3wIeB6pw7vPRM4M9/U= +github.com/aws/aws-sdk-go-v2/credentials v1.13.37/go.mod h1:ACLrdkd4CLZyXOghZ8IYumQbcooAcp2jo/s2xsFH8IM= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= @@ -155,11 +155,11 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.21.1/go.mod h1:laHbYFVzphXdCiT3gitfu github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 h1:KbGaxApdPOT2ZWqJiQY5ApnpNhUGbGTjYiKAidlFwp8= github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1/go.mod h1:+phkm4aFvcM4jbsDRGoZ+mD8MMvksHF459Xpy5Z90f0= github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 h1:oCvTFSDi67AX0pOX3PuPdGFewvLRU2zzFSrTsgURNo0= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.5/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 h1:2PylFCfKCEDv6PeSN09pC/VUiRd10wi1VfHG5FrW0/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.6/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 h1:dnInJb4S0oy8aQuri1mV6ipLlnZPfnsDNB9BGO9PDNY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 h1:pSB560BbVj9ZlJZF4WYj5zsytWHWKxg+NgyGV4B2L58= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= From 9b666a53ed4f8dd266e56bb415ad98818356363d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 17:34:57 +0000 Subject: [PATCH 46/47] Bump github.com/aws/aws-sdk-go-v2/config from 1.18.33 to 1.18.39 Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.33 to 1.18.39. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.33...config/v1.18.39) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 13 ++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 6483322c6..fcb7472b6 100644 --- a/go.mod +++ b/go.mod @@ -144,7 +144,7 @@ require ( github.com/arangodb/go-driver v1.6.0 github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go-v2 v1.21.0 - github.com/aws/aws-sdk-go-v2/config v1.18.33 + github.com/aws/aws-sdk-go-v2/config v1.18.39 github.com/aws/aws-sdk-go-v2/credentials v1.13.37 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 github.com/fluent/fluent-logger-golang v1.9.0 @@ -183,7 +183,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect diff --git a/go.sum b/go.sum index e36e2e0f0..db9d80e33 100644 --- a/go.sum +++ b/go.sum @@ -119,12 +119,10 @@ github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzR github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 h1:lN6L3LrYHeZ6xCxaIYtoWCx4GMLk4nRknsh29OMSqHY= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12/go.mod h1:TDCkEAkMTXxTs0oLBGBKpBZbk3NLh8EvAfF0Q3x8/0c= -github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= -github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= -github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= +github.com/aws/aws-sdk-go-v2/config v1.18.39 h1:oPVyh6fuu/u4OiW4qcuQyEtk7U7uuNBmHmJSLg1AJsQ= +github.com/aws/aws-sdk-go-v2/config v1.18.39/go.mod h1:+NH/ZigdPckFpgB1TRcRuWCB/Kbbvkxc/iNAKTq5RhE= github.com/aws/aws-sdk-go-v2/credentials v1.13.37 h1:BvEdm09+ZEh2XtN+PVHPcYwKY3wIeB6pw7vPRM4M9/U= github.com/aws/aws-sdk-go-v2/credentials v1.13.37/go.mod h1:ACLrdkd4CLZyXOghZ8IYumQbcooAcp2jo/s2xsFH8IM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA= @@ -135,8 +133,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFY github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39/go.mod h1:WLAW8PT7+JhjZfLSWe7WEJaJu0GNo0cKc2Zyo003RBs= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 h1:vUh7dBFNS3oFCtVv6CiYKh5hP9ls8+kIpKLeFruIBLk= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1/go.mod h1:sFMeinkhj/SZKQM8BxtvNtSPjJEo0Xrz+w3g2e4FSKI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 h1:iV/W5OMBys+66OeXJi/7xIRrKZNsu0ylsLGu+6nbmQE= @@ -154,13 +152,10 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 h1:Q01Dph/7FaB41Z7EY+SoVPa/kMpL github.com/aws/aws-sdk-go-v2/service/sns v1.21.1/go.mod h1:laHbYFVzphXdCiT3gitfuCDA2Oukrt9p40jWK7OJLgc= github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 h1:KbGaxApdPOT2ZWqJiQY5ApnpNhUGbGTjYiKAidlFwp8= github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1/go.mod h1:+phkm4aFvcM4jbsDRGoZ+mD8MMvksHF459Xpy5Z90f0= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 h1:2PylFCfKCEDv6PeSN09pC/VUiRd10wi1VfHG5FrW0/g= github.com/aws/aws-sdk-go-v2/service/sso v1.13.6/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 h1:pSB560BbVj9ZlJZF4WYj5zsytWHWKxg+NgyGV4B2L58= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= From 8908810376e671b34262295f1ad558ca43db58c2 Mon Sep 17 00:00:00 2001 From: annluda Date: Tue, 12 Sep 2023 11:06:12 +0800 Subject: [PATCH 47/47] Fix s3 api object list with params max-keys and prefix --- weed/filer/filerstore_wrapper.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index 0fac6a138..0eecdd6cb 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -292,10 +292,8 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u count := int64(0) for count < limit && len(notPrefixed) > 0 { - var isLastItemHasPrefix bool for _, entry := range notPrefixed { if strings.HasPrefix(entry.Name(), prefix) { - isLastItemHasPrefix = true count++ if !eachEntryFunc(entry) { return @@ -303,11 +301,9 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u if count >= limit { break } - } else { - isLastItemHasPrefix = false } } - if count < limit && isLastItemHasPrefix && len(notPrefixed) == int(limit) { + if count < limit && lastFileName <= prefix && len(notPrefixed) == int(limit) { notPrefixed = notPrefixed[:0] lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool { notPrefixed = append(notPrefixed, entry)