From 1a41691b4c17b36b8ad39491ce579547e1ee4c04 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Fri, 20 May 2022 14:33:47 +0800 Subject: [PATCH 1/6] exclude replication from the concurrentUploadLimitMB --- weed/server/volume_server.go | 2 ++ weed/server/volume_server_handlers.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 477a3709c..e557cf76b 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -24,6 +24,7 @@ type VolumeServer struct { inFlightDownloadDataSize int64 concurrentUploadLimit int64 concurrentDownloadLimit int64 + inFlightUploadDataLimitCond *sync.Cond inFlightDownloadDataLimitCond *sync.Cond SeedMasterNodes []pb.ServerAddress @@ -84,6 +85,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024, isHeartbeating: true, stopChan: make(chan bool), + inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)), inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), concurrentUploadLimit: concurrentUploadLimit, concurrentDownloadLimit: concurrentDownloadLimit, diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 49bc297fb..afd32fed4 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,7 +1,6 @@ package weed_server import ( - "fmt" "net/http" "strconv" "strings" @@ -60,13 +59,15 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque contentLength := getContentLength(r) // exclude the replication from the concurrentUploadLimitMB - if vs.concurrentUploadLimit != 0 && r.URL.Query().Get("type") != "replicate" && - atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { - err := fmt.Errorf("reject because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) - glog.V(1).Infof("too many requests: %v", err) - writeJsonError(w, r, http.StatusTooManyRequests, err) - return + if r.URL.Query().Get("type") != "replicate" { //Non-Replication + vs.inFlightUploadDataLimitCond.L.Lock() + for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { + glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) + vs.inFlightUploadDataLimitCond.Wait() + } + vs.inFlightUploadDataLimitCond.L.Unlock() } + atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) From 71b2e6223e07eaa5d70efdc8ccbe7f39ce6a0169 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Fri, 20 May 2022 15:19:35 +0800 Subject: [PATCH 2/6] add inFlightUploadDataLimitCond signal --- weed/server/volume_server_handlers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index afd32fed4..c199fa46a 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -71,6 +71,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) + vs.inFlightUploadDataLimitCond.Signal() }() // processs uploads From 076e48a6761396034b6c8132278330593ca698c5 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Fri, 20 May 2022 18:18:20 +0800 Subject: [PATCH 3/6] add inflight upload data wait timeout --- weed/command/server.go | 1 + weed/command/volume.go | 5 ++++- weed/server/volume_server.go | 4 ++++ weed/server/volume_server_handlers.go | 28 +++++++++++++++++++-------- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index 4b6b6c642..405d432f0 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -131,6 +131,7 @@ func init() { serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files") serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, " enable tcp port") + serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers") s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port") diff --git a/weed/command/volume.go b/weed/command/volume.go index b1455352c..158bdf162 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -65,7 +65,8 @@ type VolumeServerOptions struct { preStopSeconds *int metricsHttpPort *int // pulseSeconds *int - enableTcp *bool + enableTcp *bool + inflightUploadDataTimeout *time.Duration } func init() { @@ -96,6 +97,7 @@ func init() { v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files") v.enableTcp = cmdVolume.Flag.Bool("tcp", false, " enable tcp port") + v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers") } var cmdVolume = &Command{ @@ -244,6 +246,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v *v.fileSizeLimitMB, int64(*v.concurrentUploadLimitMB)*1024*1024, int64(*v.concurrentDownloadLimitMB)*1024*1024, + *v.inflightUploadDataTimeout, ) // starting grpc server grpcS := v.startGrpcService(volumeServer) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index e557cf76b..abb30229a 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -3,6 +3,7 @@ package weed_server import ( "net/http" "sync" + "time" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" @@ -26,6 +27,7 @@ type VolumeServer struct { concurrentDownloadLimit int64 inFlightUploadDataLimitCond *sync.Cond inFlightDownloadDataLimitCond *sync.Cond + inflightUploadDataTimeout time.Duration SeedMasterNodes []pb.ServerAddress currentMaster pb.ServerAddress @@ -61,6 +63,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, fileSizeLimitMB int, concurrentUploadLimit int64, concurrentDownloadLimit int64, + inflightUploadDataTimeout time.Duration, ) *VolumeServer { v := util.GetViper() @@ -89,6 +92,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), concurrentUploadLimit: concurrentUploadLimit, concurrentDownloadLimit: concurrentDownloadLimit, + inflightUploadDataTimeout: inflightUploadDataTimeout, } vs.SeedMasterNodes = masterNodes diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index c199fa46a..aa231e650 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,10 +1,12 @@ package weed_server import ( + "fmt" "net/http" "strconv" "strings" "sync/atomic" + "time" "github.com/chrislusf/seaweedfs/weed/util" @@ -55,22 +57,32 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque vs.guard.WhiteList(vs.DeleteHandler)(w, r) case "PUT", "POST": - // wait until in flight data is less than the limit contentLength := getContentLength(r) - + startTime := time.Now() + vs.inFlightUploadDataLimitCond.L.Lock() // exclude the replication from the concurrentUploadLimitMB - if r.URL.Query().Get("type") != "replicate" { //Non-Replication - vs.inFlightUploadDataLimitCond.L.Lock() - for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { + if r.URL.Query().Get("type") != "replicate" { + for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit { + //wait timeout + if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { + err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) + vs.inFlightUploadDataLimitCond.L.Unlock() + glog.V(1).Infof("too many requests: %v", err) + writeJsonError(w, r, http.StatusTooManyRequests, err) + return + } + glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) vs.inFlightUploadDataLimitCond.Wait() } - vs.inFlightUploadDataLimitCond.L.Unlock() } + vs.inFlightUploadDataSize += contentLength + vs.inFlightUploadDataLimitCond.L.Unlock() - atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { - atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) + vs.inFlightUploadDataLimitCond.L.Lock() + vs.inFlightUploadDataSize -= contentLength + vs.inFlightUploadDataLimitCond.L.Unlock() vs.inFlightUploadDataLimitCond.Signal() }() From f0ee3e6f2129eb2637ea0fdb18540f18147bf474 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Tue, 31 May 2022 09:40:25 +0800 Subject: [PATCH 4/6] reduce the scope of inFlightUploadDataLimitCond lock --- weed/server/volume_server_handlers.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index aa231e650..7e4c11fed 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -58,31 +58,27 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque case "PUT", "POST": contentLength := getContentLength(r) - startTime := time.Now() - vs.inFlightUploadDataLimitCond.L.Lock() // exclude the replication from the concurrentUploadLimitMB if r.URL.Query().Get("type") != "replicate" { + startTime := time.Now() + vs.inFlightUploadDataLimitCond.L.Lock() for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit { - //wait timeout + //wait timeout check if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { - err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) vs.inFlightUploadDataLimitCond.L.Unlock() + err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) glog.V(1).Infof("too many requests: %v", err) writeJsonError(w, r, http.StatusTooManyRequests, err) return } - glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) vs.inFlightUploadDataLimitCond.Wait() } - } - vs.inFlightUploadDataSize += contentLength - vs.inFlightUploadDataLimitCond.L.Unlock() - - defer func() { - vs.inFlightUploadDataLimitCond.L.Lock() - vs.inFlightUploadDataSize -= contentLength vs.inFlightUploadDataLimitCond.L.Unlock() + } + atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) + defer func() { + atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) vs.inFlightUploadDataLimitCond.Signal() }() From 3076ac101ead13d58e7b6c065af65437e3aff9cd Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Thu, 16 Jun 2022 09:58:44 +0800 Subject: [PATCH 5/6] move vs.concurrentUploadLimit != 0 out of the lock --- weed/server/volume_server_handlers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 7e4c11fed..ef37b48f5 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -59,10 +59,10 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque contentLength := getContentLength(r) // exclude the replication from the concurrentUploadLimitMB - if r.URL.Query().Get("type") != "replicate" { + if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { startTime := time.Now() vs.inFlightUploadDataLimitCond.L.Lock() - for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit { + for vs.inFlightUploadDataSize > vs.concurrentUploadLimit { //wait timeout check if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { vs.inFlightUploadDataLimitCond.L.Unlock() From df0ce31a2ee87bf4550da20cf7c1095d154b24e3 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Thu, 16 Jun 2022 14:07:11 +0800 Subject: [PATCH 6/6] add condition when inFlightUploadDataLimitCond signal --- weed/server/volume_server_handlers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index ef37b48f5..293f36f14 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -79,7 +79,9 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) - vs.inFlightUploadDataLimitCond.Signal() + if vs.concurrentUploadLimit != 0 { + vs.inFlightUploadDataLimitCond.Signal() + } }() // processs uploads