diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index bd61b047f..cafba5ce9 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -7,6 +7,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/util" "io" "mime" @@ -267,6 +268,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize if strings.Contains(post_err.Error(), "connection reset by peer") || strings.Contains(post_err.Error(), "use of closed network connection") { glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr) + stats.FilerRequestCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc() resp, post_err = HttpClient.Do(req) } } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 762e87e4b..4fb3c9609 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -14,13 +14,9 @@ import ( ) func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { - start := time.Now() - if r.Method == "OPTIONS" { - stats.FilerRequestCounter.WithLabelValues("options").Inc() OptionsHandler(w, r, false) - stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds()) return } @@ -36,12 +32,17 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { fileId = r.RequestURI[len("/?proxyChunkId="):] } if fileId != "" { - stats.FilerRequestCounter.WithLabelValues("proxy").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ChunkProxy).Inc() fs.proxyToVolumeServer(w, r, fileId) - stats.FilerRequestHistogram.WithLabelValues("proxy").Observe(time.Since(start).Seconds()) + stats.FilerRequestHistogram.WithLabelValues(stats.ChunkProxy).Observe(time.Since(start).Seconds()) return } + stats.FilerRequestCounter.WithLabelValues(r.Method).Inc() + defer func() { + stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds()) + }() + w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION) if r.Header.Get("Origin") != "" { w.Header().Set("Access-Control-Allow-Origin", "*") @@ -49,23 +50,16 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { } switch r.Method { case "GET": - stats.FilerRequestCounter.WithLabelValues("get").Inc() fs.GetOrHeadHandler(w, r) - stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) case "HEAD": - stats.FilerRequestCounter.WithLabelValues("head").Inc() fs.GetOrHeadHandler(w, r) - stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) case "DELETE": - stats.FilerRequestCounter.WithLabelValues("delete").Inc() if _, ok := r.URL.Query()["tagging"]; ok { fs.DeleteTaggingHandler(w, r) } else { fs.DeleteHandler(w, r) } - stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) case "POST", "PUT": - // wait until in flight data is less than the limit contentLength := getContentLength(r) fs.inFlightDataLimitCond.L.Lock() @@ -81,17 +75,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { }() if r.Method == "PUT" { - stats.FilerRequestCounter.WithLabelValues("put").Inc() if _, ok := r.URL.Query()["tagging"]; ok { fs.PutTaggingHandler(w, r) } else { fs.PostHandler(w, r, contentLength) } - stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds()) } else { // method == "POST" - stats.FilerRequestCounter.WithLabelValues("post").Inc() fs.PostHandler(w, r, contentLength) - stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) } } } @@ -99,12 +89,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) { start := time.Now() - + stats.FilerRequestCounter.WithLabelValues(r.Method).Inc() + defer func() { + stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds()) + }() // We handle OPTIONS first because it never should be authenticated if r.Method == "OPTIONS" { - stats.FilerRequestCounter.WithLabelValues("options").Inc() OptionsHandler(w, r, true) - stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds()) return } @@ -120,13 +111,9 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque } switch r.Method { case "GET": - stats.FilerRequestCounter.WithLabelValues("get").Inc() fs.GetOrHeadHandler(w, r) - stats.FilerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) case "HEAD": - stats.FilerRequestCounter.WithLabelValues("head").Inc() fs.GetOrHeadHandler(w, r) - stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) } } diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index 757b67dc7..0b48a9ae3 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -18,7 +18,7 @@ import ( // is empty. func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) { - stats.FilerRequestCounter.WithLabelValues("list").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.DirList).Inc() path := r.URL.Path if strings.HasSuffix(path, "/") && len(path) > 1 { diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index b3fd22a60..96f0eb81c 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -34,9 +34,11 @@ type FilerPostResult struct { func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) { - stats.FilerRequestCounter.WithLabelValues("assign").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssign).Inc() start := time.Now() - defer func() { stats.FilerRequestHistogram.WithLabelValues("assign").Observe(time.Since(start).Seconds()) }() + defer func() { + stats.FilerRequestHistogram.WithLabelValues(stats.ChunkAssign).Observe(time.Since(start).Seconds()) + }() ar, altRequest := so.ToAssignRequests(1) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 8ad3f1a1a..6e470c643 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -16,7 +16,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -34,12 +33,6 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r * chunkSize := 1024 * 1024 * maxMB - stats.FilerRequestCounter.WithLabelValues("chunk").Inc() - start := time.Now() - defer func() { - stats.FilerRequestHistogram.WithLabelValues("chunk").Observe(time.Since(start).Seconds()) - }() - var reply *FilerPostResult var err error var md5bytes []byte diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index c6d00d9ec..718d29313 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -90,8 +90,11 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque bufPool.Put(bytesBuffer) atomic.AddInt64(&bytesBufferCounter, -1) bytesBufferLimitCond.Signal() + stats.FilerRequestCounter.WithLabelValues(stats.ContentSaveToFiler).Inc() break } + } else { + stats.FilerRequestCounter.WithLabelValues(stats.AutoChunk).Inc() } wg.Add(1) @@ -138,10 +141,10 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) { - stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc() + stats.FilerRequestCounter.WithLabelValues(stats.ChunkUpload).Inc() start := time.Now() defer func() { - stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds()) + stats.FilerRequestHistogram.WithLabelValues(stats.ChunkUpload).Observe(time.Since(start).Seconds()) }() uploadOption := &operation.UploadOption{ @@ -155,7 +158,7 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil } uploadResult, err, data := operation.Upload(limitedReader, uploadOption) if uploadResult != nil && uploadResult.RetryCount > 0 { - stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount)) + stats.FilerRequestCounter.WithLabelValues(stats.ChunkUploadRetry).Add(float64(uploadResult.RetryCount)) } return uploadResult, err, data } @@ -173,6 +176,7 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch fileId, urlLocation, auth, uploadErr = fs.assignNewFileInfo(so) if uploadErr != nil { glog.V(4).Infof("retry later due to assign error: %v", uploadErr) + stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssignRetry).Inc() time.Sleep(time.Duration(i+1) * 251 * time.Millisecond) continue } @@ -181,6 +185,7 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch uploadResult, uploadErr, _ = fs.doUpload(urlLocation, dataReader, fileName, contentType, nil, auth) if uploadErr != nil { glog.V(4).Infof("retry later due to upload error: %v", uploadErr) + stats.FilerRequestCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc() time.Sleep(time.Duration(i+1) * 251 * time.Millisecond) continue } diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index 3c62dd48e..cf8b93338 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -20,14 +20,24 @@ const ( FailedToKeepConnected = "failedToKeepConnected" FailedToSend = "failedToSend" FailedToReceive = "failedToReceive" - RedirectedToleader = "redirectedToleader" + RedirectedToLeader = "redirectedToLeader" OnPeerUpdate = "onPeerUpdate" Failed = "failed" // filer handler - ErrorReadNotFound = "read.notfound" - ErrorReadInternal = "read.internalerror" - ErrorWriteEntry = "write.entry.failed" - ErrorReadCache = "read.cache.failed" - ErrorReadStream = "read.stream.failed" + DirList = "dirList" + ContentSaveToFiler = "contentSaveToFiler" + AutoChunk = "autoChunk" + ChunkProxy = "chunkProxy" + ChunkAssign = "chunkAssign" + ChunkUpload = "chunkUpload" + ChunkDoUploadRetry = "chunkDoUploadRetry" + ChunkUploadRetry = "chunkUploadRetry" + ChunkAssignRetry = "chunkAssignRetry" + ErrorReadNotFound = "read.notfound" + ErrorReadInternal = "read.internal.error" + ErrorWriteEntry = "write.entry.failed" + RepeatErrorUploadContent = "upload.content.repeat.failed" + ErrorReadCache = "read.cache.failed" + ErrorReadStream = "read.stream.failed" ) diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go index ae404f6d3..3612d978c 100644 --- a/weed/wdclient/masterclient.go +++ b/weed/wdclient/masterclient.go @@ -178,7 +178,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL if resp.VolumeLocation.Leader != "" && string(master) != resp.VolumeLocation.Leader { glog.V(0).Infof("master %v redirected to leader %v", master, resp.VolumeLocation.Leader) nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader) - stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc() return nil } //mc.vidMap = newVidMap("") @@ -203,7 +203,7 @@ func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedL if resp.VolumeLocation.Leader != "" && string(mc.currentMaster) != resp.VolumeLocation.Leader { glog.V(0).Infof("currentMaster %v redirected to leader %v", mc.currentMaster, resp.VolumeLocation.Leader) nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader) - stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc() + stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc() return nil }