From 0562fceb9925f6c2cd13b68f24d29dd1fd4c3e5e Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 23 Mar 2022 22:53:58 -0700 Subject: [PATCH] volume: fail fast if too many concurrent requests, to avoid dead lock due to replication. fix https://github.com/chrislusf/seaweedfs/issues/2755 --- weed/server/volume_server.go | 1 - weed/server/volume_server_handlers.go | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 4199ae36b..0e49af4a9 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -84,7 +84,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024, isHeartbeating: true, stopChan: make(chan bool), - inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)), inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), concurrentUploadLimit: concurrentUploadLimit, concurrentDownloadLimit: concurrentDownloadLimit, diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 510902cf0..751d3596b 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,6 +1,7 @@ package weed_server import ( + "fmt" "net/http" "strconv" "strings" @@ -51,12 +52,12 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque // wait until in flight data is less than the limit contentLength := getContentLength(r) - vs.inFlightUploadDataLimitCond.L.Lock() for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { - glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) - vs.inFlightUploadDataLimitCond.Wait() + err := fmt.Errorf("reject because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) + glog.V(1).Infof("too many requests: %v", err) + writeJsonError(w, r, http.StatusTooManyRequests, err) + return } - vs.inFlightUploadDataLimitCond.L.Unlock() atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)