add inflight upload data wait timeout

weed/command/server.go

@@ -131,6 +131,7 @@ func init() {
 	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
 	serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
 	serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, "<exprimental> enable tcp port")
+	serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
 
 	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
 	s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")

weed/command/volume.go

@@ -65,7 +65,8 @@ type VolumeServerOptions struct {
 	preStopSeconds            *int
 	metricsHttpPort           *int
 	// pulseSeconds          *int
-	enableTcp *bool
+	enableTcp                 *bool
+	inflightUploadDataTimeout *time.Duration
 }
 
 func init() {
@@ -96,6 +97,7 @@ func init() {
 	v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
 	v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<experimental> enable tcp port")
+	v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
 }
 
 var cmdVolume = &Command{
@@ -244,6 +246,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		*v.fileSizeLimitMB,
 		int64(*v.concurrentUploadLimitMB)*1024*1024,
 		int64(*v.concurrentDownloadLimitMB)*1024*1024,
+		*v.inflightUploadDataTimeout,
 	)
 	// starting grpc server
 	grpcS := v.startGrpcService(volumeServer)

weed/server/volume_server.go

@@ -3,6 +3,7 @@ package weed_server
 import (
 	"net/http"
 	"sync"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
@@ -26,6 +27,7 @@ type VolumeServer struct {
 	concurrentDownloadLimit       int64
 	inFlightUploadDataLimitCond   *sync.Cond
 	inFlightDownloadDataLimitCond *sync.Cond
+	inflightUploadDataTimeout     time.Duration
 
 	SeedMasterNodes []pb.ServerAddress
 	currentMaster   pb.ServerAddress
@@ -61,6 +63,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	fileSizeLimitMB int,
 	concurrentUploadLimit int64,
 	concurrentDownloadLimit int64,
+	inflightUploadDataTimeout time.Duration,
 ) *VolumeServer {
 
 	v := util.GetViper()
@@ -89,6 +92,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 		inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
 		concurrentUploadLimit:         concurrentUploadLimit,
 		concurrentDownloadLimit:       concurrentDownloadLimit,
+		inflightUploadDataTimeout:     inflightUploadDataTimeout,
 	}
 	vs.SeedMasterNodes = masterNodes
 

weed/server/volume_server_handlers.go

@@ -1,10 +1,12 @@
 package weed_server
 
 import (
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
 	"sync/atomic"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/util"
 
@@ -55,22 +57,32 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
 		vs.guard.WhiteList(vs.DeleteHandler)(w, r)
 	case "PUT", "POST":
 
-		// wait until in flight data is less than the limit
 		contentLength := getContentLength(r)
-
+		startTime := time.Now()
+		vs.inFlightUploadDataLimitCond.L.Lock()
 		// exclude the replication from the concurrentUploadLimitMB
-		if r.URL.Query().Get("type") != "replicate" { //Non-Replication
-			vs.inFlightUploadDataLimitCond.L.Lock()
-			for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
+		if r.URL.Query().Get("type") != "replicate" {
+			for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
+				//wait timeout
+				if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) {
+					err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
+					vs.inFlightUploadDataLimitCond.L.Unlock()
+					glog.V(1).Infof("too many requests: %v", err)
+					writeJsonError(w, r, http.StatusTooManyRequests, err)
+					return
+				}
+
 				glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
 				vs.inFlightUploadDataLimitCond.Wait()
 			}
-			vs.inFlightUploadDataLimitCond.L.Unlock()
 		}
+		vs.inFlightUploadDataSize += contentLength
+		vs.inFlightUploadDataLimitCond.L.Unlock()
 
-		atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
 		defer func() {
-			atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
+			vs.inFlightUploadDataLimitCond.L.Lock()
+			vs.inFlightUploadDataSize -= contentLength
+			vs.inFlightUploadDataLimitCond.L.Unlock()
 			vs.inFlightUploadDataLimitCond.Signal()
 		}()