From 104922a3db5068e0192ce2597cdc43714a474412 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Mon, 18 Mar 2019 00:35:15 -0700
Subject: [PATCH 01/11] text wrapping

---
 weed/server/filer_server_handlers_write_autochunk.go | 10 ++++++----
 weed/server/volume_server_handlers_write.go          |  3 +--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go
index d1e1e7a09..112f46c0c 100644
--- a/weed/server/filer_server_handlers_write_autochunk.go
+++ b/weed/server/filer_server_handlers_write_autochunk.go
@@ -19,7 +19,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, replication string, collection string, dataCenter string) bool {
+func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
+	replication string, collection string, dataCenter string) bool {
 	if r.Method != "POST" {
 		glog.V(4).Infoln("AutoChunking not supported for method", r.Method)
 		return false
@@ -64,7 +65,8 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 	return true
 }
 
-func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) {
+func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
+	contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) {
 
 	multipartReader, multipartReaderErr := r.MultipartReader()
 	if multipartReaderErr != nil {
@@ -177,8 +179,8 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r
 	return
 }
 
-func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string, auth security.EncodedJwt) (err error) {
-	err = nil
+func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request,
+	chunkBuf []byte, fileName string, contentType string, fileId string, auth security.EncodedJwt) (err error) {
 
 	ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf))
 	uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, nil, auth)
diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go
index 6b78cea40..9fb252eb7 100644
--- a/weed/server/volume_server_handlers_write.go
+++ b/weed/server/volume_server_handlers_write.go
@@ -41,8 +41,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	ret := operation.UploadResult{}
-	_, errorStatus := topology.ReplicatedWrite(vs.GetMaster(),
-		vs.store, volumeId, needle, r)
+	_, errorStatus := topology.ReplicatedWrite(vs.GetMaster(), vs.store, volumeId, needle, r)
 	httpStatus := http.StatusCreated
 	if errorStatus != "" {
 		httpStatus = http.StatusInternalServerError

From ece9d133122fd15faa4efd1b4cec34974e3b37d0 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Mon, 18 Mar 2019 09:32:21 -0700
Subject: [PATCH 02/11] volume info collect compact revision number

---
 weed/pb/master.proto           |   1 +
 weed/pb/master_pb/master.pb.go | 187 +++++++++++++++++----------------
 weed/storage/store.go          |   4 +-
 weed/storage/volume.go         |   1 +
 weed/storage/volume_info.go    |   3 +
 5 files changed, 106 insertions(+), 90 deletions(-)

diff --git a/weed/pb/master.proto b/weed/pb/master.proto
index ad530d909..ff47a16d4 100644
--- a/weed/pb/master.proto
+++ b/weed/pb/master.proto
@@ -56,6 +56,7 @@ message VolumeInformationMessage {
     uint32 replica_placement = 8;
     uint32 version = 9;
     uint32 ttl = 10;
+    uint32 compact_revision = 11;
 }
 
 message Empty {
diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go
index f24354cd7..a586d11b3 100644
--- a/weed/pb/master_pb/master.pb.go
+++ b/weed/pb/master_pb/master.pb.go
@@ -190,6 +190,7 @@ type VolumeInformationMessage struct {
 	ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"`
 	Version          uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"`
 	Ttl              uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
+	CompactRevision  uint32 `protobuf:"varint,11,opt,name=compact_revision,json=compactRevision" json:"compact_revision,omitempty"`
 }
 
 func (m *VolumeInformationMessage) Reset()                    { *m = VolumeInformationMessage{} }
@@ -267,6 +268,13 @@ func (m *VolumeInformationMessage) GetTtl() uint32 {
 	return 0
 }
 
+func (m *VolumeInformationMessage) GetCompactRevision() uint32 {
+	if m != nil {
+		return m.CompactRevision
+	}
+	return 0
+}
+
 type Empty struct {
 }
 
@@ -1428,93 +1436,94 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("master.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1394 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0x4b, 0x6f, 0xdb, 0xc6,
-	0x13, 0x37, 0xa9, 0x87, 0xc5, 0xd1, 0xc3, 0xd2, 0xda, 0x49, 0x18, 0xe5, 0x9f, 0x44, 0x61, 0x2e,
-	0xfa, 0xbf, 0x8c, 0xd4, 0x3d, 0xf4, 0xd0, 0x16, 0x41, 0xe2, 0x38, 0x68, 0x10, 0xb7, 0x49, 0xe8,
-	0x24, 0x05, 0x0a, 0x14, 0xea, 0x9a, 0x1c, 0xbb, 0x84, 0x29, 0x92, 0x25, 0x57, 0x8e, 0x95, 0x4b,
-	0x2f, 0x3d, 0x16, 0xed, 0xa1, 0xdf, 0xa7, 0x97, 0xf6, 0xd6, 0x8f, 0xd2, 0x5b, 0xef, 0x05, 0x8a,
-	0x7d, 0x90, 0x5a, 0x52, 0xb2, 0x13, 0x14, 0xe8, 0x21, 0xb7, 0xdd, 0x99, 0xd9, 0xdd, 0xe1, 0xef,
-	0x37, 0x2f, 0x09, 0x3a, 0x53, 0x9a, 0x31, 0x4c, 0xb7, 0x93, 0x34, 0x66, 0x31, 0xb1, 0xe4, 0x6e,
-	0x92, 0x1c, 0x3a, 0xbf, 0x9b, 0x60, 0x7d, 0x82, 0x34, 0x65, 0x87, 0x48, 0x19, 0xe9, 0x81, 0x19,
-	0x24, 0xb6, 0x31, 0x32, 0xc6, 0x96, 0x6b, 0x06, 0x09, 0x21, 0x50, 0x4f, 0xe2, 0x94, 0xd9, 0xe6,
-	0xc8, 0x18, 0x77, 0x5d, 0xb1, 0x26, 0xd7, 0x01, 0x92, 0xd9, 0x61, 0x18, 0x78, 0x93, 0x59, 0x1a,
-	0xda, 0x35, 0x61, 0x6b, 0x49, 0xc9, 0x8b, 0x34, 0x24, 0x63, 0xe8, 0x4f, 0xe9, 0xd9, 0xe4, 0x34,
-	0x0e, 0x67, 0x53, 0x9c, 0x78, 0xf1, 0x2c, 0x62, 0x76, 0x5d, 0x1c, 0xef, 0x4d, 0xe9, 0xd9, 0x4b,
-	0x21, 0xde, 0xe5, 0x52, 0x32, 0xe2, 0x5e, 0x9d, 0x4d, 0x8e, 0x82, 0x10, 0x27, 0x27, 0x38, 0xb7,
-	0x1b, 0x23, 0x63, 0x5c, 0x77, 0x61, 0x4a, 0xcf, 0x1e, 0x06, 0x21, 0x3e, 0xc6, 0x39, 0xb9, 0x09,
-	0x6d, 0x9f, 0x32, 0x3a, 0xf1, 0x30, 0x62, 0x98, 0xda, 0x4d, 0xf1, 0x16, 0x70, 0xd1, 0xae, 0x90,
-	0x70, 0xff, 0x52, 0xea, 0x9d, 0xd8, 0xeb, 0x42, 0x23, 0xd6, 0xdc, 0x3f, 0xea, 0x4f, 0x83, 0x68,
-	0x22, 0x3c, 0x6f, 0x89, 0xa7, 0x2d, 0x21, 0x79, 0xca, 0xdd, 0xff, 0x18, 0xd6, 0xa5, 0x6f, 0x99,
-	0x6d, 0x8d, 0x6a, 0xe3, 0xf6, 0xce, 0xed, 0xed, 0x02, 0x8d, 0x6d, 0xe9, 0xde, 0xa3, 0xe8, 0x28,
-	0x4e, 0xa7, 0x94, 0x05, 0x71, 0xf4, 0x29, 0x66, 0x19, 0x3d, 0x46, 0x37, 0x3f, 0x43, 0xae, 0x42,
-	0x2b, 0xc2, 0x57, 0x93, 0xd3, 0xc0, 0xcf, 0x6c, 0x18, 0xd5, 0xc6, 0x5d, 0x77, 0x3d, 0xc2, 0x57,
-	0x2f, 0x03, 0x3f, 0x23, 0xb7, 0xa0, 0xe3, 0x63, 0x88, 0x0c, 0x7d, 0xa9, 0x6e, 0x0b, 0x75, 0x5b,
-	0xc9, 0xb8, 0x89, 0xf3, 0x02, 0x06, 0x05, 0xd8, 0x2e, 0x66, 0x49, 0x1c, 0x65, 0x48, 0xc6, 0xb0,
-	0x21, 0x6f, 0x3f, 0x08, 0x5e, 0xe3, 0x7e, 0x30, 0x0d, 0x98, 0x60, 0xa0, 0xee, 0x56, 0xc5, 0xe4,
-	0x32, 0x34, 0x43, 0xa4, 0x3e, 0xa6, 0x0a, 0x76, 0xb5, 0x73, 0x7e, 0x35, 0xc1, 0x3e, 0xcf, 0x75,
-	0xc1, 0xa9, 0x2f, 0x6e, 0xec, 0xba, 0x66, 0xe0, 0x73, 0xcc, 0xb2, 0xe0, 0x35, 0x0a, 0x4e, 0xeb,
-	0xae, 0x58, 0x93, 0x1b, 0x00, 0x5e, 0x1c, 0x86, 0xe8, 0xf1, 0x83, 0xea, 0x72, 0x4d, 0xc2, 0x31,
-	0x15, 0x34, 0x2d, 0xe8, 0xac, 0xbb, 0x16, 0x97, 0x48, 0x26, 0x8b, 0x2f, 0x57, 0x06, 0x92, 0x49,
-	0xf5, 0xe5, 0xd2, 0xe4, 0x7f, 0x40, 0x72, 0x70, 0x0e, 0xe7, 0x85, 0x61, 0x53, 0x18, 0xf6, 0x95,
-	0xe6, 0xfe, 0x3c, 0xb7, 0xbe, 0x06, 0x56, 0x8a, 0xd4, 0x9f, 0xc4, 0x51, 0x38, 0x17, 0xe4, 0xb6,
-	0xdc, 0x16, 0x17, 0x3c, 0x89, 0xc2, 0x39, 0xf9, 0x2f, 0x0c, 0x52, 0x4c, 0xc2, 0xc0, 0xa3, 0x93,
-	0x24, 0xa4, 0x1e, 0x4e, 0x31, 0xca, 0x79, 0xee, 0x2b, 0xc5, 0xd3, 0x5c, 0x4e, 0x6c, 0x58, 0x3f,
-	0xc5, 0x34, 0xe3, 0x9f, 0x65, 0x09, 0x93, 0x7c, 0x4b, 0xfa, 0x50, 0x63, 0x2c, 0xb4, 0x41, 0x48,
-	0xf9, 0xd2, 0x59, 0x87, 0xc6, 0xde, 0x34, 0x61, 0x73, 0xe7, 0x67, 0x03, 0x36, 0x0e, 0x66, 0x09,
-	0xa6, 0xf7, 0xc3, 0xd8, 0x3b, 0xd9, 0x3b, 0x63, 0x29, 0x25, 0x4f, 0xa0, 0x87, 0x29, 0xcd, 0x66,
-	0x29, 0xf7, 0xdd, 0x0f, 0xa2, 0x63, 0x01, 0x69, 0x7b, 0x67, 0xac, 0x85, 0x4f, 0xe5, 0xcc, 0xf6,
-	0x9e, 0x3c, 0xb0, 0x2b, 0xec, 0xdd, 0x2e, 0xea, 0xdb, 0xe1, 0x17, 0xd0, 0x2d, 0xe9, 0x39, 0x31,
-	0x3c, 0xb4, 0x15, 0x55, 0x62, 0xcd, 0x19, 0x4f, 0x68, 0x1a, 0xb0, 0xb9, 0x4a, 0x41, 0xb5, 0xe3,
-	0x84, 0xa8, 0x0c, 0xe3, 0x91, 0x56, 0x13, 0x91, 0x66, 0x49, 0xc9, 0x23, 0x3f, 0x73, 0xfe, 0x0d,
-	0x9b, 0xbb, 0x61, 0x80, 0x11, 0xdb, 0x0f, 0x32, 0x86, 0x91, 0x8b, 0xdf, 0xcc, 0x30, 0x63, 0xfc,
-	0x85, 0x88, 0x4e, 0x51, 0x25, 0xb8, 0x58, 0x3b, 0xdf, 0x42, 0x4f, 0x86, 0xce, 0x7e, 0xec, 0x89,
-	0xb8, 0xe1, 0xc0, 0xf0, 0xcc, 0x96, 0x46, 0x7c, 0x59, 0x49, 0x79, 0xb3, 0x9a, 0xf2, 0x7a, 0x4e,
-	0xd4, 0x2e, 0xce, 0x89, 0xfa, 0x72, 0x4e, 0x3c, 0x87, 0xcd, 0xfd, 0x38, 0x3e, 0x99, 0x25, 0xd2,
-	0x8d, 0xdc, 0xd7, 0xf2, 0x17, 0x1a, 0xa3, 0x1a, 0x7f, 0xb3, 0xf8, 0xc2, 0x4a, 0xc4, 0x9a, 0xd5,
-	0x88, 0x75, 0xfe, 0x30, 0x60, 0xab, 0x7c, 0xad, 0xca, 0xb6, 0xaf, 0x60, 0xb3, 0xb8, 0x77, 0x12,
-	0xaa, 0x6f, 0x96, 0x0f, 0xb4, 0x77, 0xee, 0x68, 0x64, 0xae, 0x3a, 0x9d, 0x17, 0x08, 0x3f, 0x07,
-	0xcb, 0x1d, 0x9c, 0x56, 0x24, 0xd9, 0xf0, 0x0c, 0xfa, 0x55, 0x33, 0x1e, 0xd0, 0xc5, 0xab, 0x0a,
-	0xd9, 0x56, 0x7e, 0x92, 0xbc, 0x07, 0xd6, 0xc2, 0x11, 0x53, 0x38, 0xb2, 0x59, 0x72, 0x44, 0xbd,
-	0xb5, 0xb0, 0x22, 0x5b, 0xd0, 0xc0, 0x34, 0x8d, 0xf3, 0x42, 0x20, 0x37, 0xce, 0x87, 0xd0, 0xfa,
-	0xdb, 0x2c, 0x3a, 0xbf, 0x19, 0xd0, 0xbd, 0x97, 0x65, 0xc1, 0x71, 0x11, 0x2e, 0x5b, 0xd0, 0x90,
-	0x69, 0x2a, 0xcb, 0x91, 0xdc, 0x90, 0x11, 0xb4, 0x55, 0x96, 0x69, 0xd0, 0xeb, 0xa2, 0x37, 0x56,
-	0x13, 0x95, 0x79, 0x75, 0xe9, 0x1a, 0x63, 0x61, 0xb5, 0xd0, 0x37, 0xce, 0x2d, 0xf4, 0x4d, 0xad,
-	0xd0, 0x5f, 0x03, 0x4b, 0x1c, 0x8a, 0x62, 0x1f, 0x55, 0x07, 0x68, 0x71, 0xc1, 0x67, 0xb1, 0x8f,
-	0xce, 0x4f, 0x06, 0xf4, 0xf2, 0xaf, 0x51, 0xcc, 0xf7, 0xa1, 0x76, 0x54, 0xa0, 0xcf, 0x97, 0x39,
-	0x46, 0xe6, 0x79, 0x18, 0x2d, 0x35, 0xb7, 0x02, 0x91, 0xba, 0x8e, 0x48, 0x41, 0x46, 0x43, 0x23,
-	0x83, 0xbb, 0x4c, 0x67, 0xec, 0xeb, 0xdc, 0x65, 0xbe, 0x76, 0x8e, 0x61, 0x70, 0xc0, 0x28, 0x0b,
-	0x32, 0x16, 0x78, 0x59, 0x0e, 0x73, 0x05, 0x50, 0xe3, 0x4d, 0x80, 0x9a, 0xe7, 0x01, 0x5a, 0x2b,
-	0x00, 0x75, 0x7e, 0x31, 0x80, 0xe8, 0x2f, 0x29, 0x08, 0xfe, 0x81, 0xa7, 0x38, 0x64, 0x2c, 0x66,
-	0x34, 0x9c, 0x88, 0xae, 0xa2, 0x7a, 0x83, 0x90, 0xf0, 0xc6, 0xc5, 0x59, 0x9a, 0x65, 0xe8, 0x4b,
-	0xad, 0x6c, 0x0c, 0x2d, 0x2e, 0x10, 0xca, 0x72, 0x5f, 0x69, 0x56, 0xfa, 0x8a, 0x73, 0x0f, 0xda,
-	0x07, 0x2c, 0x4e, 0xe9, 0x31, 0x3e, 0x9f, 0x27, 0x6f, 0xe3, 0xbd, 0xf2, 0xce, 0x5c, 0x00, 0x31,
-	0x02, 0xd8, 0x5d, 0x78, 0xbf, 0xaa, 0x00, 0x5e, 0x81, 0x4b, 0x0b, 0x0b, 0x5e, 0x2f, 0x15, 0x2f,
-	0xce, 0x33, 0xb8, 0x5c, 0x55, 0x28, 0x18, 0x3f, 0x80, 0xf6, 0x02, 0x92, 0xbc, 0x76, 0x5c, 0xd2,
-	0x52, 0x76, 0x71, 0xce, 0xd5, 0x2d, 0x9d, 0xff, 0xc3, 0x95, 0x85, 0xea, 0x81, 0x28, 0x82, 0x17,
-	0xd5, 0xe6, 0x21, 0xd8, 0xcb, 0xe6, 0xd2, 0x07, 0xe7, 0x47, 0x13, 0x3a, 0x0f, 0x54, 0xb4, 0xf3,
-	0xae, 0xaf, 0xf5, 0x79, 0x4b, 0xf4, 0xf9, 0x5b, 0xd0, 0x29, 0x0d, 0x61, 0xb2, 0xdf, 0xb7, 0x4f,
-	0xb5, 0x09, 0x6c, 0xd5, 0xac, 0x56, 0x13, 0x66, 0xd5, 0x59, 0xed, 0x3f, 0x30, 0x38, 0x4a, 0x11,
-	0x97, 0xc7, 0xba, 0xba, 0xbb, 0xc1, 0x15, 0xba, 0xed, 0x36, 0x6c, 0x52, 0x8f, 0x05, 0xa7, 0x15,
-	0x6b, 0xc9, 0xfd, 0x40, 0xaa, 0x74, 0xfb, 0x87, 0x85, 0xa3, 0x41, 0x74, 0x14, 0x67, 0x76, 0xf3,
-	0xed, 0xc7, 0x32, 0xf5, 0x35, 0x5c, 0x93, 0x39, 0xdf, 0x99, 0xd0, 0x72, 0xa9, 0x77, 0xf2, 0x6e,
-	0xa3, 0x71, 0x17, 0x36, 0x8a, 0xaa, 0x56, 0x02, 0xe4, 0x8a, 0x06, 0x88, 0x4e, 0xbc, 0xdb, 0xf5,
-	0xb5, 0x5d, 0xe6, 0xfc, 0x69, 0x40, 0xef, 0x41, 0x51, 0x39, 0xdf, 0x6d, 0x30, 0x76, 0x00, 0x78,
-	0xa9, 0x2f, 0xe1, 0xa0, 0xb7, 0xc6, 0x9c, 0x6e, 0xd7, 0x4a, 0xd5, 0x2a, 0x73, 0x7e, 0x30, 0xa1,
-	0xf3, 0x3c, 0x4e, 0xe2, 0x30, 0x3e, 0x9e, 0xbf, 0xdb, 0x5f, 0xbf, 0x07, 0x03, 0xad, 0x2b, 0x96,
-	0x40, 0xb8, 0x5a, 0x09, 0x86, 0x05, 0xd9, 0xee, 0x86, 0x5f, 0xda, 0x67, 0xce, 0x26, 0x0c, 0xd4,
-	0x84, 0xa7, 0x15, 0x37, 0x17, 0x88, 0x2e, 0x54, 0x85, 0xed, 0x23, 0xe8, 0x32, 0x05, 0x9d, 0x78,
-	0x4e, 0xcd, 0xb8, 0x7a, 0xe8, 0xe9, 0xd0, 0xba, 0x1d, 0xa6, 0xed, 0x76, 0xbe, 0x6f, 0xc0, 0xfa,
-	0x01, 0xd2, 0x57, 0x88, 0x3e, 0x79, 0x04, 0xdd, 0x03, 0x8c, 0xfc, 0xc5, 0x4f, 0xcb, 0x2d, 0xed,
-	0x8e, 0x42, 0x3a, 0xfc, 0xd7, 0x2a, 0x69, 0x51, 0xe3, 0xd6, 0xc6, 0xc6, 0x1d, 0x83, 0x3c, 0x85,
-	0xee, 0x63, 0xc4, 0x64, 0x37, 0x8e, 0x22, 0xf4, 0x18, 0xfa, 0xe4, 0x86, 0x5e, 0x69, 0x97, 0xc7,
-	0xdc, 0xe1, 0xd5, 0xa5, 0xd2, 0x91, 0x4f, 0x45, 0xea, 0xc6, 0x67, 0xd0, 0xd1, 0xa7, 0xbb, 0xd2,
-	0x85, 0x2b, 0x66, 0xd1, 0xe1, 0xcd, 0x37, 0x8c, 0x85, 0xce, 0x1a, 0xb9, 0x0b, 0x4d, 0x39, 0x6e,
-	0x10, 0x5b, 0x33, 0x2e, 0xcd, 0x53, 0x25, 0xbf, 0xca, 0xb3, 0x89, 0xb3, 0x46, 0x1e, 0x03, 0x2c,
-	0x1a, 0x36, 0xd1, 0x71, 0x59, 0x9a, 0x18, 0x86, 0xd7, 0xcf, 0xd1, 0x16, 0x97, 0x7d, 0x0e, 0xbd,
-	0x72, 0xeb, 0x22, 0xa3, 0x95, 0xdd, 0x49, 0x8b, 0x88, 0xe1, 0xad, 0x0b, 0x2c, 0x8a, 0x8b, 0xbf,
-	0x84, 0x7e, 0xb5, 0x23, 0x11, 0x67, 0xe5, 0xc1, 0x52, 0x77, 0x1b, 0xde, 0xbe, 0xd0, 0x46, 0x07,
-	0x61, 0x11, 0x95, 0x25, 0x10, 0x96, 0x22, 0xb8, 0x04, 0xc2, 0x72, 0x28, 0x3b, 0x6b, 0x87, 0x4d,
-	0xf1, 0x67, 0xc7, 0xfb, 0x7f, 0x05, 0x00, 0x00, 0xff, 0xff, 0xbf, 0x50, 0x28, 0x66, 0xfc, 0x10,
-	0x00, 0x00,
+	// 1416 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6f, 0xdc, 0x44,
+	0x14, 0x8f, 0xbd, 0x1f, 0x59, 0xbf, 0xfd, 0xc8, 0xee, 0x24, 0x6d, 0xdd, 0x2d, 0x6d, 0xb7, 0xee,
+	0x65, 0xcb, 0x47, 0x54, 0xc2, 0x81, 0x03, 0xa0, 0xaa, 0x4d, 0x53, 0x51, 0x35, 0xd0, 0xd6, 0x69,
+	0x8b, 0x84, 0x84, 0xcc, 0xc4, 0x7e, 0x09, 0x56, 0xbc, 0xb6, 0xb1, 0x67, 0xb7, 0xd9, 0x5e, 0xb8,
+	0x70, 0x44, 0x70, 0xe0, 0xff, 0xe1, 0xc2, 0x91, 0x3f, 0x85, 0x03, 0x12, 0x77, 0x24, 0x34, 0xe3,
+	0xb1, 0x77, 0xec, 0xdd, 0xa4, 0x15, 0x12, 0x87, 0xde, 0x3c, 0xbf, 0xf7, 0x66, 0xe6, 0xcd, 0xef,
+	0x7d, 0xee, 0x42, 0x67, 0x42, 0x53, 0x86, 0xc9, 0x76, 0x9c, 0x44, 0x2c, 0x22, 0x46, 0xb6, 0x72,
+	0xe2, 0x43, 0xeb, 0x4f, 0x1d, 0x8c, 0xcf, 0x91, 0x26, 0xec, 0x10, 0x29, 0x23, 0x3d, 0xd0, 0xfd,
+	0xd8, 0xd4, 0x46, 0xda, 0xd8, 0xb0, 0x75, 0x3f, 0x26, 0x04, 0xea, 0x71, 0x94, 0x30, 0x53, 0x1f,
+	0x69, 0xe3, 0xae, 0x2d, 0xbe, 0xc9, 0x55, 0x80, 0x78, 0x7a, 0x18, 0xf8, 0xae, 0x33, 0x4d, 0x02,
+	0xb3, 0x26, 0x74, 0x8d, 0x0c, 0x79, 0x9e, 0x04, 0x64, 0x0c, 0xfd, 0x09, 0x3d, 0x75, 0x66, 0x51,
+	0x30, 0x9d, 0xa0, 0xe3, 0x46, 0xd3, 0x90, 0x99, 0x75, 0xb1, 0xbd, 0x37, 0xa1, 0xa7, 0x2f, 0x04,
+	0xbc, 0xcb, 0x51, 0x32, 0xe2, 0x56, 0x9d, 0x3a, 0x47, 0x7e, 0x80, 0xce, 0x09, 0xce, 0xcd, 0xc6,
+	0x48, 0x1b, 0xd7, 0x6d, 0x98, 0xd0, 0xd3, 0x07, 0x7e, 0x80, 0x8f, 0x70, 0x4e, 0xae, 0x43, 0xdb,
+	0xa3, 0x8c, 0x3a, 0x2e, 0x86, 0x0c, 0x13, 0xb3, 0x29, 0xee, 0x02, 0x0e, 0xed, 0x0a, 0x84, 0xdb,
+	0x97, 0x50, 0xf7, 0xc4, 0x5c, 0x17, 0x12, 0xf1, 0xcd, 0xed, 0xa3, 0xde, 0xc4, 0x0f, 0x1d, 0x61,
+	0x79, 0x4b, 0x5c, 0x6d, 0x08, 0xe4, 0x09, 0x37, 0xff, 0x33, 0x58, 0xcf, 0x6c, 0x4b, 0x4d, 0x63,
+	0x54, 0x1b, 0xb7, 0x77, 0x6e, 0x6e, 0x17, 0x6c, 0x6c, 0x67, 0xe6, 0x3d, 0x0c, 0x8f, 0xa2, 0x64,
+	0x42, 0x99, 0x1f, 0x85, 0x5f, 0x60, 0x9a, 0xd2, 0x63, 0xb4, 0xf3, 0x3d, 0xe4, 0x32, 0xb4, 0x42,
+	0x7c, 0xe9, 0xcc, 0x7c, 0x2f, 0x35, 0x61, 0x54, 0x1b, 0x77, 0xed, 0xf5, 0x10, 0x5f, 0xbe, 0xf0,
+	0xbd, 0x94, 0xdc, 0x80, 0x8e, 0x87, 0x01, 0x32, 0xf4, 0x32, 0x71, 0x5b, 0x88, 0xdb, 0x12, 0xe3,
+	0x2a, 0xd6, 0x73, 0x18, 0x14, 0x64, 0xdb, 0x98, 0xc6, 0x51, 0x98, 0x22, 0x19, 0xc3, 0x46, 0x76,
+	0xfa, 0x81, 0xff, 0x0a, 0xf7, 0xfd, 0x89, 0xcf, 0x84, 0x07, 0xea, 0x76, 0x15, 0x26, 0x17, 0xa1,
+	0x19, 0x20, 0xf5, 0x30, 0x91, 0xb4, 0xcb, 0x95, 0xf5, 0x97, 0x0e, 0xe6, 0x59, 0xa6, 0x0b, 0x9f,
+	0x7a, 0xe2, 0xc4, 0xae, 0xad, 0xfb, 0x1e, 0xe7, 0x2c, 0xf5, 0x5f, 0xa1, 0xf0, 0x69, 0xdd, 0x16,
+	0xdf, 0xe4, 0x1a, 0x80, 0x1b, 0x05, 0x01, 0xba, 0x7c, 0xa3, 0x3c, 0x5c, 0x41, 0x38, 0xa7, 0xc2,
+	0x4d, 0x0b, 0x77, 0xd6, 0x6d, 0x83, 0x23, 0x99, 0x27, 0x8b, 0x97, 0x4b, 0x85, 0xcc, 0x93, 0xf2,
+	0xe5, 0x99, 0xca, 0xfb, 0x40, 0x72, 0x72, 0x0e, 0xe7, 0x85, 0x62, 0x53, 0x28, 0xf6, 0xa5, 0xe4,
+	0xde, 0x3c, 0xd7, 0xbe, 0x02, 0x46, 0x82, 0xd4, 0x73, 0xa2, 0x30, 0x98, 0x0b, 0xe7, 0xb6, 0xec,
+	0x16, 0x07, 0x1e, 0x87, 0xc1, 0x9c, 0xbc, 0x07, 0x83, 0x04, 0xe3, 0xc0, 0x77, 0xa9, 0x13, 0x07,
+	0xd4, 0xc5, 0x09, 0x86, 0xb9, 0x9f, 0xfb, 0x52, 0xf0, 0x24, 0xc7, 0x89, 0x09, 0xeb, 0x33, 0x4c,
+	0x52, 0xfe, 0x2c, 0x43, 0xa8, 0xe4, 0x4b, 0xd2, 0x87, 0x1a, 0x63, 0x81, 0x09, 0x02, 0xe5, 0x9f,
+	0xe4, 0x16, 0xf4, 0xdd, 0x68, 0x12, 0x53, 0x97, 0x39, 0x09, 0xce, 0x7c, 0xb1, 0xa9, 0x2d, 0xc4,
+	0x1b, 0x12, 0xb7, 0x25, 0x6c, 0xad, 0x43, 0x63, 0x6f, 0x12, 0xb3, 0xb9, 0xf5, 0x9b, 0x06, 0x1b,
+	0x07, 0xd3, 0x18, 0x93, 0x7b, 0x41, 0xe4, 0x9e, 0xec, 0x9d, 0xb2, 0x84, 0x92, 0xc7, 0xd0, 0xc3,
+	0x84, 0xa6, 0xd3, 0x84, 0x3f, 0xd3, 0xf3, 0xc3, 0x63, 0xc1, 0x7e, 0x7b, 0x67, 0xac, 0x44, 0x5a,
+	0x65, 0xcf, 0xf6, 0x5e, 0xb6, 0x61, 0x57, 0xe8, 0xdb, 0x5d, 0x54, 0x97, 0xc3, 0xaf, 0xa1, 0x5b,
+	0x92, 0x73, 0x1f, 0xf2, 0x2c, 0x90, 0x5e, 0x15, 0xdf, 0x3c, 0x38, 0x62, 0x9a, 0xf8, 0x6c, 0x2e,
+	0xb3, 0x55, 0xae, 0xb8, 0xef, 0x64, 0x32, 0xf2, 0xa0, 0xac, 0x89, 0xa0, 0x34, 0x32, 0xe4, 0xa1,
+	0x97, 0x5a, 0xb7, 0x60, 0x73, 0x37, 0xf0, 0x31, 0x64, 0xfb, 0x7e, 0xca, 0x30, 0xb4, 0xf1, 0xfb,
+	0x29, 0xa6, 0x8c, 0xdf, 0x10, 0xd2, 0x09, 0xca, 0x5a, 0x20, 0xbe, 0xad, 0x1f, 0xa0, 0x97, 0x45,
+	0xd9, 0x7e, 0xe4, 0x8a, 0x10, 0xe3, 0x1c, 0xf2, 0x22, 0x90, 0x29, 0xf1, 0xcf, 0x4a, 0x75, 0xd0,
+	0xab, 0xd5, 0x41, 0x4d, 0x9f, 0xda, 0xf9, 0xe9, 0x53, 0x5f, 0x4e, 0x9f, 0x67, 0xb0, 0xb9, 0x1f,
+	0x45, 0x27, 0xd3, 0x38, 0x33, 0x23, 0xb7, 0xb5, 0xfc, 0x42, 0x6d, 0x54, 0xe3, 0x77, 0x16, 0x2f,
+	0xac, 0x04, 0xb7, 0x5e, 0x0d, 0x6e, 0xeb, 0x6f, 0x0d, 0xb6, 0xca, 0xc7, 0xca, 0xc4, 0xfc, 0x16,
+	0x36, 0x8b, 0x73, 0x9d, 0x40, 0xbe, 0x39, 0xbb, 0xa0, 0xbd, 0x73, 0x5b, 0x71, 0xe6, 0xaa, 0xdd,
+	0x79, 0x2d, 0xf1, 0x72, 0xb2, 0xec, 0xc1, 0xac, 0x82, 0xa4, 0xc3, 0x53, 0xe8, 0x57, 0xd5, 0x78,
+	0xec, 0x17, 0xb7, 0x4a, 0x66, 0x5b, 0xf9, 0x4e, 0xf2, 0x21, 0x18, 0x0b, 0x43, 0x74, 0x61, 0xc8,
+	0x66, 0xc9, 0x10, 0x79, 0xd7, 0x42, 0x8b, 0x6c, 0x41, 0x03, 0x93, 0x24, 0xca, 0x6b, 0x46, 0xb6,
+	0xb0, 0x3e, 0x81, 0xd6, 0x7f, 0xf6, 0xa2, 0xf5, 0x87, 0x06, 0xdd, 0xbb, 0x69, 0xea, 0x1f, 0x17,
+	0xe1, 0xb2, 0x05, 0x8d, 0x2c, 0xa3, 0xb3, 0xca, 0x95, 0x2d, 0xc8, 0x08, 0xda, 0x32, 0x21, 0x15,
+	0xea, 0x55, 0xe8, 0xb5, 0x85, 0x47, 0x26, 0x69, 0x3d, 0x33, 0x8d, 0x27, 0x69, 0xa5, 0x27, 0x34,
+	0xce, 0xec, 0x09, 0x4d, 0xa5, 0x27, 0x5c, 0x01, 0x43, 0x6c, 0x0a, 0x23, 0x0f, 0x65, 0xb3, 0x68,
+	0x71, 0xe0, 0xcb, 0xc8, 0x43, 0xeb, 0x57, 0x0d, 0x7a, 0xf9, 0x6b, 0xa4, 0xe7, 0xfb, 0x50, 0x3b,
+	0x2a, 0xd8, 0xe7, 0x9f, 0x39, 0x47, 0xfa, 0x59, 0x1c, 0x2d, 0xf5, 0xc1, 0x82, 0x91, 0xba, 0xca,
+	0x48, 0xe1, 0x8c, 0x86, 0xe2, 0x0c, 0x6e, 0x32, 0x9d, 0xb2, 0xef, 0x72, 0x93, 0xf9, 0xb7, 0x75,
+	0x0c, 0x83, 0x03, 0x46, 0x99, 0x9f, 0x32, 0xdf, 0x4d, 0x73, 0x9a, 0x2b, 0x84, 0x6a, 0xaf, 0x23,
+	0x54, 0x3f, 0x8b, 0xd0, 0x5a, 0x41, 0xa8, 0xf5, 0xbb, 0x06, 0x44, 0xbd, 0x49, 0x52, 0xf0, 0x3f,
+	0x5c, 0xc5, 0x29, 0x63, 0x11, 0xa3, 0x81, 0x23, 0x1a, 0x90, 0x6c, 0x23, 0x02, 0xe1, 0x3d, 0x8e,
+	0x7b, 0x69, 0x9a, 0xa2, 0x97, 0x49, 0xb3, 0x1e, 0xd2, 0xe2, 0x80, 0x10, 0x96, 0x5b, 0x50, 0xb3,
+	0xd2, 0x82, 0xac, 0xbb, 0xd0, 0x3e, 0x60, 0x51, 0x42, 0x8f, 0xf1, 0xd9, 0x3c, 0x7e, 0x13, 0xeb,
+	0xa5, 0x75, 0xfa, 0x82, 0x88, 0x11, 0xc0, 0xee, 0xc2, 0xfa, 0x55, 0x05, 0xf0, 0x12, 0x5c, 0x58,
+	0x68, 0xf0, 0x7a, 0x29, 0xfd, 0x62, 0x3d, 0x85, 0x8b, 0x55, 0x81, 0xa4, 0xf1, 0x63, 0x68, 0x2f,
+	0x28, 0xc9, 0x6b, 0xc7, 0x05, 0x25, 0x65, 0x17, 0xfb, 0x6c, 0x55, 0xd3, 0xfa, 0x00, 0x2e, 0x2d,
+	0x44, 0xf7, 0x45, 0x11, 0x3c, 0xaf, 0x36, 0x0f, 0xc1, 0x5c, 0x56, 0xcf, 0x6c, 0xb0, 0x7e, 0xd1,
+	0xa1, 0x73, 0x5f, 0x46, 0x3b, 0x1f, 0x10, 0x94, 0x91, 0xc0, 0x10, 0x23, 0xc1, 0x0d, 0xe8, 0x94,
+	0xe6, 0xb5, 0x6c, 0x34, 0x68, 0xcf, 0x94, 0x61, 0x6d, 0xd5, 0x58, 0x57, 0x13, 0x6a, 0xd5, 0xb1,
+	0xee, 0x5d, 0x18, 0x1c, 0x25, 0x88, 0xcb, 0x13, 0x60, 0xdd, 0xde, 0xe0, 0x02, 0x55, 0x77, 0x1b,
+	0x36, 0xa9, 0xcb, 0xfc, 0x59, 0x45, 0x3b, 0xf3, 0xfd, 0x20, 0x13, 0xa9, 0xfa, 0x0f, 0x0a, 0x43,
+	0xfd, 0xf0, 0x28, 0x4a, 0xcd, 0xe6, 0x9b, 0x4f, 0x70, 0xf2, 0x35, 0x5c, 0x92, 0x5a, 0x3f, 0xea,
+	0xd0, 0xb2, 0xa9, 0x7b, 0xf2, 0x76, 0xb3, 0x71, 0x07, 0x36, 0x8a, 0xaa, 0x56, 0x22, 0xe4, 0x92,
+	0x42, 0x88, 0xea, 0x78, 0xbb, 0xeb, 0x29, 0xab, 0xd4, 0xfa, 0x47, 0x83, 0xde, 0xfd, 0xa2, 0x72,
+	0xbe, 0xdd, 0x64, 0xec, 0x00, 0xf0, 0x52, 0x5f, 0xe2, 0x41, 0x6d, 0x8d, 0xb9, 0xbb, 0x6d, 0x23,
+	0x91, 0x5f, 0xa9, 0xf5, 0xb3, 0x0e, 0x9d, 0x67, 0x51, 0x1c, 0x05, 0xd1, 0xf1, 0xfc, 0xed, 0x7e,
+	0xfd, 0x1e, 0x0c, 0x94, 0xae, 0x58, 0x22, 0xe1, 0x72, 0x25, 0x18, 0x16, 0xce, 0xb6, 0x37, 0xbc,
+	0xd2, 0x3a, 0xb5, 0x36, 0x61, 0x20, 0x27, 0x3c, 0xa5, 0xb8, 0xd9, 0x40, 0x54, 0x50, 0x16, 0xb6,
+	0x4f, 0xa1, 0xcb, 0x24, 0x75, 0xe2, 0x3a, 0x39, 0xe3, 0xaa, 0xa1, 0xa7, 0x52, 0x6b, 0x77, 0x98,
+	0xb2, 0xda, 0xf9, 0xa9, 0x01, 0xeb, 0x07, 0x48, 0x5f, 0x22, 0x7a, 0xe4, 0x21, 0x74, 0x0f, 0x30,
+	0xf4, 0x16, 0xbf, 0x42, 0xb7, 0x94, 0x33, 0x0a, 0x74, 0xf8, 0xce, 0x2a, 0xb4, 0xa8, 0x71, 0x6b,
+	0x63, 0xed, 0xb6, 0x46, 0x9e, 0x40, 0xf7, 0x11, 0x62, 0xbc, 0x1b, 0x85, 0x21, 0xba, 0x0c, 0x3d,
+	0x72, 0x4d, 0xad, 0xb4, 0xcb, 0x63, 0xee, 0xf0, 0xf2, 0x52, 0xe9, 0xc8, 0xa7, 0x22, 0x79, 0xe2,
+	0x53, 0xe8, 0xa8, 0xd3, 0x5d, 0xe9, 0xc0, 0x15, 0xb3, 0xe8, 0xf0, 0xfa, 0x6b, 0xc6, 0x42, 0x6b,
+	0x8d, 0xdc, 0x81, 0x66, 0x36, 0x6e, 0x10, 0x53, 0x51, 0x2e, 0xcd, 0x53, 0x25, 0xbb, 0xca, 0xb3,
+	0x89, 0xb5, 0x46, 0x1e, 0x01, 0x2c, 0x1a, 0x36, 0x51, 0x79, 0x59, 0x9a, 0x18, 0x86, 0x57, 0xcf,
+	0x90, 0x16, 0x87, 0x7d, 0x05, 0xbd, 0x72, 0xeb, 0x22, 0xa3, 0x95, 0xdd, 0x49, 0x89, 0x88, 0xe1,
+	0x8d, 0x73, 0x34, 0x8a, 0x83, 0xbf, 0x81, 0x7e, 0xb5, 0x23, 0x11, 0x6b, 0xe5, 0xc6, 0x52, 0x77,
+	0x1b, 0xde, 0x3c, 0x57, 0x47, 0x25, 0x61, 0x11, 0x95, 0x25, 0x12, 0x96, 0x22, 0xb8, 0x44, 0xc2,
+	0x72, 0x28, 0x5b, 0x6b, 0x87, 0x4d, 0xf1, 0xbf, 0xc8, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff,
+	0x88, 0xac, 0x76, 0x9c, 0x27, 0x11, 0x00, 0x00,
 }
diff --git a/weed/storage/store.go b/weed/storage/store.go
index a29f3c163..8d4d9c55e 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -121,7 +121,9 @@ func (s *Store) Status() []*VolumeInfo {
 				DeleteCount:      v.nm.DeletedCount(),
 				DeletedByteCount: v.nm.DeletedSize(),
 				ReadOnly:         v.readOnly,
-				Ttl:              v.Ttl}
+				Ttl:              v.Ttl,
+				CompactRevision:  uint32(v.CompactRevision),
+			}
 			stats = append(stats, s)
 		}
 		location.RUnlock()
diff --git a/weed/storage/volume.go b/weed/storage/volume.go
index 5cec0c5ed..22acf1653 100644
--- a/weed/storage/volume.go
+++ b/weed/storage/volume.go
@@ -148,5 +148,6 @@ func (v *Volume) ToVolumeInformationMessage() *master_pb.VolumeInformationMessag
 		ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
 		Version:          uint32(v.Version()),
 		Ttl:              v.Ttl.ToUint32(),
+		CompactRevision:  uint32(v.SuperBlock.CompactRevision),
 	}
 }
diff --git a/weed/storage/volume_info.go b/weed/storage/volume_info.go
index 7afad412d..f5ddeca14 100644
--- a/weed/storage/volume_info.go
+++ b/weed/storage/volume_info.go
@@ -18,6 +18,7 @@ type VolumeInfo struct {
 	DeleteCount      int
 	DeletedByteCount uint64
 	ReadOnly         bool
+	CompactRevision  uint32
 }
 
 func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err error) {
@@ -30,6 +31,7 @@ func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err er
 		DeletedByteCount: m.DeletedByteCount,
 		ReadOnly:         m.ReadOnly,
 		Version:          Version(m.Version),
+		CompactRevision:  m.CompactRevision,
 	}
 	rp, e := NewReplicaPlacementFromByte(byte(m.ReplicaPlacement))
 	if e != nil {
@@ -57,6 +59,7 @@ func (vi VolumeInfo) ToVolumeInformationMessage() *master_pb.VolumeInformationMe
 		ReplicaPlacement: uint32(vi.ReplicaPlacement.Byte()),
 		Version:          uint32(vi.Version),
 		Ttl:              vi.Ttl.ToUint32(),
+		CompactRevision:  vi.CompactRevision,
 	}
 }
 

From eb4a54d9fe865eafb4de12db96b3e668b333fd1a Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Tue, 19 Mar 2019 05:19:37 -0700
Subject: [PATCH 03/11] refactoring

---
 weed/server/filer_grpc_server.go         |  7 +++++-
 weed/shell/command_collection_list.go    | 10 ++++++--
 weed/shell/command_volume_list.go        | 10 +++++---
 weed/wdclient/masterclient.go            |  6 +++++
 weed/wdclient/masterclient_collection.go | 31 ------------------------
 5 files changed, 26 insertions(+), 38 deletions(-)
 delete mode 100644 weed/wdclient/masterclient_collection.go

diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go
index 13c66543b..6f7cf1ad6 100644
--- a/weed/server/filer_grpc_server.go
+++ b/weed/server/filer_grpc_server.go
@@ -238,7 +238,12 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol
 
 func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.DeleteCollectionRequest) (resp *filer_pb.DeleteCollectionResponse, err error) {
 
-	err = fs.filer.MasterClient.CollectionDelete(ctx, req.GetCollection())
+	err = fs.filer.MasterClient.WithClient(ctx, func(ctx context.Context, client master_pb.SeaweedClient) error {
+		_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
+			Name: req.GetCollection(),
+		})
+		return err
+	})
 
 	return &filer_pb.DeleteCollectionResponse{}, err
 }
diff --git a/weed/shell/command_collection_list.go b/weed/shell/command_collection_list.go
index 34a406d67..0797e56fb 100644
--- a/weed/shell/command_collection_list.go
+++ b/weed/shell/command_collection_list.go
@@ -3,6 +3,7 @@ package shell
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"io"
 )
 
@@ -21,9 +22,14 @@ func (c *commandCollectionList) Help() string {
 	return "# list all collections"
 }
 
-func (c *commandCollectionList) Do(args []string, commandEnv *commandEnv, writer io.Writer) error {
+func (c *commandCollectionList) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
 
-	resp, err := commandEnv.masterClient.CollectionList(context.Background())
+	var resp *master_pb.CollectionListResponse
+
+	err = commandEnv.masterClient.WithClient(context.Background(), func(ctx context.Context, client master_pb.SeaweedClient) error {
+		resp, err = client.CollectionList(ctx, &master_pb.CollectionListRequest{})
+		return err
+	})
 
 	if err != nil {
 		return err
diff --git a/weed/shell/command_volume_list.go b/weed/shell/command_volume_list.go
index 971e18f10..52ac4865a 100644
--- a/weed/shell/command_volume_list.go
+++ b/weed/shell/command_volume_list.go
@@ -22,16 +22,18 @@ func (c *commandVolumeList) Help() string {
 	return "# list all volumes"
 }
 
-func (c *commandVolumeList) Do(args []string, commandEnv *commandEnv, writer io.Writer) error {
-
-	resp, err := commandEnv.masterClient.VolumeList(context.Background())
+func (c *commandVolumeList) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
 
+	var resp *master_pb.VolumeListResponse
+	err = commandEnv.masterClient.WithClient(context.Background(), func(ctx context.Context, client master_pb.SeaweedClient) error {
+		resp, err = client.VolumeList(ctx, &master_pb.VolumeListRequest{})
+		return err
+	})
 	if err != nil {
 		return err
 	}
 
 	writeTopologyInfo(writer, resp.TopologyInfo)
-
 	return nil
 }
 
diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go
index b3b277c74..5f147e594 100644
--- a/weed/wdclient/masterclient.go
+++ b/weed/wdclient/masterclient.go
@@ -116,3 +116,9 @@ func withMasterClient(ctx context.Context, master string, grpcDialOption grpc.Di
 
 	return fn(ctx, client)
 }
+
+func (mc *MasterClient) WithClient(ctx context.Context, fn func(ctx context.Context, client master_pb.SeaweedClient) error) error {
+	return withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
+		return fn(ctx, client)
+	})
+}
diff --git a/weed/wdclient/masterclient_collection.go b/weed/wdclient/masterclient_collection.go
deleted file mode 100644
index 1a9215a7e..000000000
--- a/weed/wdclient/masterclient_collection.go
+++ /dev/null
@@ -1,31 +0,0 @@
-package wdclient
-
-import (
-	"context"
-	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
-)
-
-func (mc *MasterClient) CollectionDelete(ctx context.Context, collection string) error {
-	return withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
-		_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
-			Name: collection,
-		})
-		return err
-	})
-}
-
-func (mc *MasterClient) CollectionList(ctx context.Context) (resp *master_pb.CollectionListResponse, err error) {
-	err = withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
-		resp, err = client.CollectionList(ctx, &master_pb.CollectionListRequest{})
-		return err
-	})
-	return
-}
-
-func (mc *MasterClient) VolumeList(ctx context.Context) (resp *master_pb.VolumeListResponse, err error) {
-	err = withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
-		resp, err = client.VolumeList(ctx, &master_pb.VolumeListRequest{})
-		return err
-	})
-	return
-}

From 916b809c086e94208eabe2af1086bcb1289ed298 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Tue, 19 Mar 2019 05:34:43 -0700
Subject: [PATCH 04/11] add error checking

---
 weed/storage/needle_read_write.go | 39 +++++++++++++++++++++++--------
 weed/storage/volume_vacuum.go     |  5 +++-
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go
index e69d70dc3..4e6bd9cc7 100644
--- a/weed/storage/needle_read_write.go
+++ b/weed/storage/needle_read_write.go
@@ -182,10 +182,10 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
 	case Version1:
 		n.Data = bytes[NeedleEntrySize : NeedleEntrySize+size]
 	case Version2, Version3:
-		n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
+		err = n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
 	}
-	if size == 0 {
-		return nil
+	if size == 0 || err != nil {
+		return err
 	}
 	checksum := util.BytesToUint32(bytes[NeedleEntrySize+size : NeedleEntrySize+size+NeedleChecksumSize])
 	newChecksum := NewCRC(n.Data)
@@ -206,15 +206,15 @@ func (n *Needle) ParseNeedleHeader(bytes []byte) {
 	n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize : NeedleEntrySize])
 }
 
-func (n *Needle) readNeedleDataVersion2(bytes []byte) {
+var ErrIndexOutOfRange = fmt.Errorf("index out of range")
+
+func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
 	index, lenBytes := 0, len(bytes)
 	if index < lenBytes {
 		n.DataSize = util.BytesToUint32(bytes[index : index+4])
 		index = index + 4
-		if int(n.DataSize)+index > lenBytes {
-			// this if clause is due to bug #87 and #93, fixed in v0.69
-			// remove this clause later
-			return
+		if int(n.DataSize)+index >= lenBytes {
+			return ErrIndexOutOfRange
 		}
 		n.Data = bytes[index : index+int(n.DataSize)]
 		index = index + int(n.DataSize)
@@ -224,30 +224,49 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) {
 	if index < lenBytes && n.HasName() {
 		n.NameSize = uint8(bytes[index])
 		index = index + 1
+		if int(n.NameSize)+index >= lenBytes {
+			return ErrIndexOutOfRange
+		}
 		n.Name = bytes[index : index+int(n.NameSize)]
 		index = index + int(n.NameSize)
 	}
 	if index < lenBytes && n.HasMime() {
 		n.MimeSize = uint8(bytes[index])
 		index = index + 1
+		if int(n.MimeSize)+index >= lenBytes {
+			return ErrIndexOutOfRange
+		}
 		n.Mime = bytes[index : index+int(n.MimeSize)]
 		index = index + int(n.MimeSize)
 	}
 	if index < lenBytes && n.HasLastModifiedDate() {
+		if LastModifiedBytesLength+index >= lenBytes {
+			return ErrIndexOutOfRange
+		}
 		n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
 		index = index + LastModifiedBytesLength
 	}
 	if index < lenBytes && n.HasTtl() {
+		if TtlBytesLength+index >= lenBytes {
+			return ErrIndexOutOfRange
+		}
 		n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
 		index = index + TtlBytesLength
 	}
 	if index < lenBytes && n.HasPairs() {
+		if 2+index >= lenBytes {
+			return ErrIndexOutOfRange
+		}
 		n.PairsSize = util.BytesToUint16(bytes[index : index+2])
 		index += 2
+		if int(n.PairsSize)+index >= lenBytes {
+			return ErrIndexOutOfRange
+		}
 		end := index + int(n.PairsSize)
 		n.Pairs = bytes[index:end]
 		index = end
 	}
+	return nil
 }
 
 func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength int64, err error) {
@@ -300,11 +319,11 @@ func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyL
 		if _, err = r.ReadAt(bytes, offset); err != nil {
 			return
 		}
-		n.readNeedleDataVersion2(bytes[0:n.Size])
+		err = n.readNeedleDataVersion2(bytes[0:n.Size])
 		n.Checksum = NewCRC(n.Data)
 
 		if version == Version3 {
-			tsOffset := n.Size+NeedleChecksumSize
+			tsOffset := n.Size + NeedleChecksumSize
 			n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize])
 		}
 	default:
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index b4870423c..63aada2c2 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -335,7 +335,10 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
 		}
 
 		n := new(Needle)
-		n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version())
+		err :=n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version())
+		if err != nil {
+			return nil
+		}
 
 		if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) {
 			return nil

From da871896c345372780c35de7364b42d910935c53 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Tue, 19 Mar 2019 05:47:41 -0700
Subject: [PATCH 05/11] weed filer: set grpc port to port + 10000

---
 weed/command/mount.go                       |  7 +------
 weed/command/mount_std.go                   |  2 +-
 weed/command/s3.go                          |  2 +-
 weed/operation/grpc_client.go               |  2 +-
 weed/server/raft_server.go                  |  4 ++--
 weed/server/volume_grpc_client_to_master.go |  2 +-
 weed/storage/volume_vacuum.go               |  2 +-
 weed/util/grpc_client_server.go             | 11 ++++-------
 weed/wdclient/masterclient.go               |  2 +-
 9 files changed, 13 insertions(+), 21 deletions(-)

diff --git a/weed/command/mount.go b/weed/command/mount.go
index 760c68e40..ec790c999 100644
--- a/weed/command/mount.go
+++ b/weed/command/mount.go
@@ -8,7 +8,6 @@ import (
 
 type MountOptions struct {
 	filer              *string
-	filerGrpcPort      *int
 	filerMountRootPath *string
 	dir                *string
 	dirListingLimit    *int
@@ -29,7 +28,6 @@ var (
 func init() {
 	cmdMount.Run = runMount // break init cycle
 	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
-	mountOptions.filerGrpcPort = cmdMount.Flag.Int("filer.grpc.port", 0, "filer grpc server listen port, default to http port + 10000")
 	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
 	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
 	mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 100000, "limit directory listing size")
@@ -61,7 +59,7 @@ var cmdMount = &Command{
   `,
 }
 
-func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress string, err error) {
+func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
 	hostnameAndPort := strings.Split(filer, ":")
 	if len(hostnameAndPort) != 2 {
 		return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort)
@@ -73,9 +71,6 @@ func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress
 	}
 
 	filerGrpcPort := int(filerPort) + 10000
-	if optionalGrpcPort != 0 {
-		filerGrpcPort = optionalGrpcPort
-	}
 
 	return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
 }
diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go
index 2b274e200..c047b94c3 100644
--- a/weed/command/mount_std.go
+++ b/weed/command/mount_std.go
@@ -87,7 +87,7 @@ func runMount(cmd *Command, args []string) bool {
 		c.Close()
 	})
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer, *mountOptions.filerGrpcPort)
+	filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer)
 	if err != nil {
 		glog.Fatal(err)
 		return false
diff --git a/weed/command/s3.go b/weed/command/s3.go
index a54ddd2f7..9aa9f8e2f 100644
--- a/weed/command/s3.go
+++ b/weed/command/s3.go
@@ -51,7 +51,7 @@ func runS3(cmd *Command, args []string) bool {
 
 	weed_server.LoadConfiguration("security", false)
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
+	filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer)
 	if err != nil {
 		glog.Fatal(err)
 		return false
diff --git a/weed/operation/grpc_client.go b/weed/operation/grpc_client.go
index eb97f5ce1..ea7a82044 100644
--- a/weed/operation/grpc_client.go
+++ b/weed/operation/grpc_client.go
@@ -42,7 +42,7 @@ func withMasterServerClient(masterServer string, grpcDialOption grpc.DialOption,
 
 	ctx := context.Background()
 
-	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer, 0)
+	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer)
 	if parseErr != nil {
 		return fmt.Errorf("failed to parse master grpc %v", masterServer)
 	}
diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go
index 4be13810f..88320ed98 100644
--- a/weed/server/raft_server.go
+++ b/weed/server/raft_server.go
@@ -61,7 +61,7 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr st
 	s.raftServer.Start()
 
 	for _, peer := range s.peers {
-		s.raftServer.AddPeer(peer, util.ServerToGrpcAddress(peer, 19333))
+		s.raftServer.AddPeer(peer, util.ServerToGrpcAddress(peer))
 	}
 
 	s.GrpcServer = raft.NewGrpcServer(s.raftServer)
@@ -72,7 +72,7 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr st
 
 		_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
 			Name:             s.raftServer.Name(),
-			ConnectionString: util.ServerToGrpcAddress(s.serverAddr, 19333),
+			ConnectionString: util.ServerToGrpcAddress(s.serverAddr),
 		})
 
 		if err != nil {
diff --git a/weed/server/volume_grpc_client_to_master.go b/weed/server/volume_grpc_client_to_master.go
index f6ed8ee23..94e99c8f6 100644
--- a/weed/server/volume_grpc_client_to_master.go
+++ b/weed/server/volume_grpc_client_to_master.go
@@ -31,7 +31,7 @@ func (vs *VolumeServer) heartbeat() {
 			if newLeader != "" {
 				master = newLeader
 			}
-			masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master, 0)
+			masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master)
 			if parseErr != nil {
 				glog.V(0).Infof("failed to parse master grpc %v", masterGrpcAddress)
 				continue
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index 63aada2c2..b29e15843 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -335,7 +335,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
 		}
 
 		n := new(Needle)
-		err :=n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version())
+		err := n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version())
 		if err != nil {
 			return nil
 		}
diff --git a/weed/util/grpc_client_server.go b/weed/util/grpc_client_server.go
index 361d245b8..e5993aeab 100644
--- a/weed/util/grpc_client_server.go
+++ b/weed/util/grpc_client_server.go
@@ -81,7 +81,7 @@ func WithCachedGrpcClient(ctx context.Context, fn func(*grpc.ClientConn) error,
 	return err
 }
 
-func ParseServerToGrpcAddress(server string, optionalGrpcPort int) (serverGrpcAddress string, err error) {
+func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
 	hostnameAndPort := strings.Split(server, ":")
 	if len(hostnameAndPort) != 2 {
 		return "", fmt.Errorf("server should have hostname:port format: %v", hostnameAndPort)
@@ -93,22 +93,19 @@ func ParseServerToGrpcAddress(server string, optionalGrpcPort int) (serverGrpcAd
 	}
 
 	grpcPort := int(port) + 10000
-	if optionalGrpcPort != 0 {
-		grpcPort = optionalGrpcPort
-	}
 
 	return fmt.Sprintf("%s:%d", hostnameAndPort[0], grpcPort), nil
 }
 
-func ServerToGrpcAddress(server string, defaultGrpcPort int) (serverGrpcAddress string) {
+func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
 	hostnameAndPort := strings.Split(server, ":")
 	if len(hostnameAndPort) != 2 {
-		return fmt.Sprintf("%s:%d", server, defaultGrpcPort)
+		return fmt.Sprintf("unexpected server address: %s", server)
 	}
 
 	port, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
 	if parseErr != nil {
-		return fmt.Sprintf("%s:%d", hostnameAndPort[0], defaultGrpcPort)
+		return fmt.Sprintf("failed to parse port for %s:%s", hostnameAndPort[0], hostnameAndPort[1])
 	}
 
 	grpcPort := int(port) + 10000
diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go
index 5f147e594..29fe50d80 100644
--- a/weed/wdclient/masterclient.go
+++ b/weed/wdclient/masterclient.go
@@ -101,7 +101,7 @@ func (mc *MasterClient) tryAllMasters() {
 
 func withMasterClient(ctx context.Context, master string, grpcDialOption grpc.DialOption, fn func(ctx context.Context, client master_pb.SeaweedClient) error) error {
 
-	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master, 0)
+	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master)
 	if parseErr != nil {
 		return fmt.Errorf("failed to parse master grpc %v", master)
 	}

From cea5c53bf7943f7735d74c1e9f568ba633474e5d Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Tue, 19 Mar 2019 20:56:27 -0700
Subject: [PATCH 06/11] fix length checking

fix https://github.com/chrislusf/seaweedfs/issues/890
---
 weed/storage/needle_read_write.go | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go
index 4e6bd9cc7..0a73b2977 100644
--- a/weed/storage/needle_read_write.go
+++ b/weed/storage/needle_read_write.go
@@ -206,15 +206,13 @@ func (n *Needle) ParseNeedleHeader(bytes []byte) {
 	n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize : NeedleEntrySize])
 }
 
-var ErrIndexOutOfRange = fmt.Errorf("index out of range")
-
 func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
 	index, lenBytes := 0, len(bytes)
 	if index < lenBytes {
 		n.DataSize = util.BytesToUint32(bytes[index : index+4])
 		index = index + 4
-		if int(n.DataSize)+index >= lenBytes {
-			return ErrIndexOutOfRange
+		if int(n.DataSize)+index > lenBytes {
+			return fmt.Errorf("index out of range %d", 1)
 		}
 		n.Data = bytes[index : index+int(n.DataSize)]
 		index = index + int(n.DataSize)
@@ -224,8 +222,8 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
 	if index < lenBytes && n.HasName() {
 		n.NameSize = uint8(bytes[index])
 		index = index + 1
-		if int(n.NameSize)+index >= lenBytes {
-			return ErrIndexOutOfRange
+		if int(n.NameSize)+index > lenBytes {
+			return fmt.Errorf("index out of range %d", 2)
 		}
 		n.Name = bytes[index : index+int(n.NameSize)]
 		index = index + int(n.NameSize)
@@ -233,34 +231,34 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
 	if index < lenBytes && n.HasMime() {
 		n.MimeSize = uint8(bytes[index])
 		index = index + 1
-		if int(n.MimeSize)+index >= lenBytes {
-			return ErrIndexOutOfRange
+		if int(n.MimeSize)+index > lenBytes {
+			return fmt.Errorf("index out of range %d", 3)
 		}
 		n.Mime = bytes[index : index+int(n.MimeSize)]
 		index = index + int(n.MimeSize)
 	}
 	if index < lenBytes && n.HasLastModifiedDate() {
-		if LastModifiedBytesLength+index >= lenBytes {
-			return ErrIndexOutOfRange
+		if LastModifiedBytesLength+index > lenBytes {
+			return fmt.Errorf("index out of range %d", 4)
 		}
 		n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
 		index = index + LastModifiedBytesLength
 	}
 	if index < lenBytes && n.HasTtl() {
-		if TtlBytesLength+index >= lenBytes {
-			return ErrIndexOutOfRange
+		if TtlBytesLength+index > lenBytes {
+			return fmt.Errorf("index out of range %d", 5)
 		}
 		n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
 		index = index + TtlBytesLength
 	}
 	if index < lenBytes && n.HasPairs() {
-		if 2+index >= lenBytes {
-			return ErrIndexOutOfRange
+		if 2+index > lenBytes {
+			return fmt.Errorf("index out of range %d", 6)
 		}
 		n.PairsSize = util.BytesToUint16(bytes[index : index+2])
 		index += 2
-		if int(n.PairsSize)+index >= lenBytes {
-			return ErrIndexOutOfRange
+		if int(n.PairsSize)+index > lenBytes {
+			return fmt.Errorf("index out of range %d", 7)
 		}
 		end := index + int(n.PairsSize)
 		n.Pairs = bytes[index:end]

From f3d316a846473fe8f33a90172291608d764a264d Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Tue, 19 Mar 2019 21:58:00 -0700
Subject: [PATCH 07/11] weed shell: adding file system disk usage fs.du

fix https://github.com/chrislusf/seaweedfs/issues/889

echo "fs.du http://localhost:8888/some/path" | weed shell
---
 weed/shell/command_fs_du.go | 137 ++++++++++++++++++++++++++++++++++++
 weed/shell/commands.go      |   1 +
 weed/shell/shell_liner.go   |   1 +
 3 files changed, 139 insertions(+)
 create mode 100644 weed/shell/command_fs_du.go

diff --git a/weed/shell/command_fs_du.go b/weed/shell/command_fs_du.go
new file mode 100644
index 000000000..cdc9d98ef
--- /dev/null
+++ b/weed/shell/command_fs_du.go
@@ -0,0 +1,137 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"io"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+func init() {
+	commands = append(commands, &commandFsDu{})
+}
+
+type commandFsDu struct {
+}
+
+func (c *commandFsDu) Name() string {
+	return "fs.du"
+}
+
+func (c *commandFsDu) Help() string {
+	return "http://<filer_server>:<port>/dir[/file] # show disk usage"
+}
+
+func (c *commandFsDu) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
+
+	filerServer, filerPort, path, err := parseFilerUrl(args[0])
+	if err != nil {
+		return err
+	}
+
+	dir, name := filer2.FullPath(path).DirAndName()
+	if strings.HasSuffix(path, "/") {
+		if path == "/"{
+			dir, name = "/", ""
+		}else{
+			dir, name = path[0 : len(path)-1], ""
+		}
+	}
+
+	ctx := context.Background()
+
+	return commandEnv.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
+
+		_, _, err = paginateDirectory(ctx, writer, client, dir, name, 1000)
+
+		return err
+
+	})
+
+}
+
+func paginateDirectory(ctx context.Context, writer io.Writer, client filer_pb.SeaweedFilerClient, dir, name string, paginateSize int) (blockCount uint64, byteCount uint64, err error) {
+
+	paginatedCount := -1
+	startFromFileName := ""
+
+	for paginatedCount == -1 || paginatedCount == paginateSize {
+		resp, listErr := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
+			Directory:          dir,
+			Prefix:             name,
+			StartFromFileName:  startFromFileName,
+			InclusiveStartFrom: false,
+			Limit:              uint32(paginateSize),
+		})
+		if listErr != nil {
+			err = listErr
+			return
+		}
+
+		paginatedCount = len(resp.Entries)
+
+		for _, entry := range resp.Entries {
+			if entry.IsDirectory {
+				subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
+				if dir == "/" {
+					subDir = "/" + entry.Name
+				}
+				numBlock, numByte, err := paginateDirectory(ctx, writer, client, subDir, "", paginateSize)
+				if err == nil {
+					blockCount += numBlock
+					byteCount += numByte
+				}
+			} else {
+				blockCount += uint64(len(entry.Chunks))
+				byteCount += filer2.TotalSize(entry.Chunks)
+			}
+			startFromFileName = entry.Name
+
+			if name != "" && !entry.IsDirectory {
+				fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s/%s\n", blockCount, byteCount, dir, name)
+			}
+		}
+	}
+
+	if name == "" {
+		fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir)
+	}
+
+	return
+
+}
+
+func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
+	if strings.HasPrefix(entryPath, "http") {
+		var u *url.URL
+		u, err = url.Parse(entryPath)
+		if err != nil {
+			return
+		}
+		filerServer = u.Hostname()
+		portString := u.Port()
+		if portString != "" {
+			filerPort, err = strconv.ParseInt(portString, 10, 32)
+		}
+		path = u.Path
+	} else {
+		err = fmt.Errorf("path should have full url http://<filer_server>:<port>/path/to/dirOrFile : %s", entryPath)
+	}
+	return
+}
+
+func (env *commandEnv) withFilerClient(ctx context.Context, filerServer string, filerPort int64, fn func(filer_pb.SeaweedFilerClient) error) error {
+
+	filerGrpcAddress := fmt.Sprintf("%s:%d", filerServer, filerPort+10000)
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, filerGrpcAddress, env.option.GrpcDialOption)
+
+}
diff --git a/weed/shell/commands.go b/weed/shell/commands.go
index 4df70ff55..280900c80 100644
--- a/weed/shell/commands.go
+++ b/weed/shell/commands.go
@@ -14,6 +14,7 @@ type ShellOptions struct {
 type commandEnv struct {
 	env          map[string]string
 	masterClient *wdclient.MasterClient
+	option       ShellOptions
 }
 
 type command interface {
diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go
index cd015fe85..096532fdf 100644
--- a/weed/shell/shell_liner.go
+++ b/weed/shell/shell_liner.go
@@ -36,6 +36,7 @@ func RunShell(options ShellOptions) {
 		env: make(map[string]string),
 		masterClient: wdclient.NewMasterClient(context.Background(),
 			options.GrpcDialOption, "shell", strings.Split(*options.Masters, ",")),
+		option: options,
 	}
 
 	go commandEnv.masterClient.KeepConnectedToMaster()

From 5ae4b963a47a98426392b5434a9c3e26ad550e27 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Tue, 19 Mar 2019 22:20:14 -0700
Subject: [PATCH 08/11] avoid using global rand

---
 weed/wdclient/vid_map.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/weed/wdclient/vid_map.go b/weed/wdclient/vid_map.go
index aef29f56f..64c24a809 100644
--- a/weed/wdclient/vid_map.go
+++ b/weed/wdclient/vid_map.go
@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 )
@@ -19,11 +20,13 @@ type Location struct {
 type vidMap struct {
 	sync.RWMutex
 	vid2Locations map[uint32][]Location
+	r             *rand.Rand
 }
 
 func newVidMap() vidMap {
 	return vidMap{
 		vid2Locations: make(map[uint32][]Location),
+		r:             rand.New(rand.NewSource(time.Now().UnixNano())),
 	}
 }
 
@@ -39,7 +42,7 @@ func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrl string, err error
 		return "", fmt.Errorf("volume %d not found", id)
 	}
 
-	return locations[rand.Intn(len(locations))].Url, nil
+	return locations[vc.r.Intn(len(locations))].Url, nil
 }
 
 func (vc *vidMap) LookupFileId(fileId string) (fullUrl string, err error) {

From 88ab932f7d439715698183fbdfbd4ade71596b67 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Tue, 19 Mar 2019 23:01:23 -0700
Subject: [PATCH 09/11] refactoring function parameter

---
 weed/server/filer_grpc_server.go      | 2 +-
 weed/shell/command_collection_list.go | 4 ++--
 weed/shell/command_volume_list.go     | 3 ++-
 weed/wdclient/masterclient.go         | 4 ++--
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go
index 6f7cf1ad6..c9cb6dbe7 100644
--- a/weed/server/filer_grpc_server.go
+++ b/weed/server/filer_grpc_server.go
@@ -238,7 +238,7 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol
 
 func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.DeleteCollectionRequest) (resp *filer_pb.DeleteCollectionResponse, err error) {
 
-	err = fs.filer.MasterClient.WithClient(ctx, func(ctx context.Context, client master_pb.SeaweedClient) error {
+	err = fs.filer.MasterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
 		_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
 			Name: req.GetCollection(),
 		})
diff --git a/weed/shell/command_collection_list.go b/weed/shell/command_collection_list.go
index 0797e56fb..0db74ef20 100644
--- a/weed/shell/command_collection_list.go
+++ b/weed/shell/command_collection_list.go
@@ -25,8 +25,8 @@ func (c *commandCollectionList) Help() string {
 func (c *commandCollectionList) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
 
 	var resp *master_pb.CollectionListResponse
-
-	err = commandEnv.masterClient.WithClient(context.Background(), func(ctx context.Context, client master_pb.SeaweedClient) error {
+	ctx := context.Background()
+	err = commandEnv.masterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
 		resp, err = client.CollectionList(ctx, &master_pb.CollectionListRequest{})
 		return err
 	})
diff --git a/weed/shell/command_volume_list.go b/weed/shell/command_volume_list.go
index 52ac4865a..5be5be569 100644
--- a/weed/shell/command_volume_list.go
+++ b/weed/shell/command_volume_list.go
@@ -25,7 +25,8 @@ func (c *commandVolumeList) Help() string {
 func (c *commandVolumeList) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
 
 	var resp *master_pb.VolumeListResponse
-	err = commandEnv.masterClient.WithClient(context.Background(), func(ctx context.Context, client master_pb.SeaweedClient) error {
+	ctx := context.Background()
+	err = commandEnv.masterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
 		resp, err = client.VolumeList(ctx, &master_pb.VolumeListRequest{})
 		return err
 	})
diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go
index 29fe50d80..794471f7b 100644
--- a/weed/wdclient/masterclient.go
+++ b/weed/wdclient/masterclient.go
@@ -117,8 +117,8 @@ func withMasterClient(ctx context.Context, master string, grpcDialOption grpc.Di
 	return fn(ctx, client)
 }
 
-func (mc *MasterClient) WithClient(ctx context.Context, fn func(ctx context.Context, client master_pb.SeaweedClient) error) error {
+func (mc *MasterClient) WithClient(ctx context.Context, fn func(client master_pb.SeaweedClient) error) error {
 	return withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
-		return fn(ctx, client)
+		return fn(client)
 	})
 }

From 45a52b17fd83a9ffcda494de5cd800a93de97c9c Mon Sep 17 00:00:00 2001
From: chenwanli <chenwanli@sensetime.com>
Date: Wed, 20 Mar 2019 16:38:11 +0800
Subject: [PATCH 10/11] benchmark: add replication flag

---
 weed/command/benchmark.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go
index 44601e567..aa54946a7 100644
--- a/weed/command/benchmark.go
+++ b/weed/command/benchmark.go
@@ -36,6 +36,7 @@ type BenchmarkOptions struct {
 	read             *bool
 	sequentialRead   *bool
 	collection       *string
+	replication      *string
 	cpuprofile       *string
 	maxCpu           *int
 	grpcDialOption   grpc.DialOption
@@ -61,6 +62,7 @@ func init() {
 	b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
 	b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
 	b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
+	b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
 	b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
 	b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	sharedBytes = make([]byte, 1024)
@@ -228,8 +230,9 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 			MimeType: "image/bench", // prevent gzip benchmark content
 		}
 		ar := &operation.VolumeAssignRequest{
-			Count:      1,
-			Collection: *b.collection,
+			Count:       1,
+			Collection:  *b.collection,
+			Replication: *b.replication,
 		}
 		if assignResult, err := operation.Assign(masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
 			fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection

From 977b30e992108e480c883cdd461bf0c8f6598f25 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@uber.com>
Date: Wed, 20 Mar 2019 10:13:51 -0700
Subject: [PATCH 11/11] weed filer: http HEAD response header add ETag

fix https://github.com/chrislusf/seaweedfs/issues/892
---
 weed/server/filer_server_handlers_read.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go
index 4d1f41fd4..89d47b0b8 100644
--- a/weed/server/filer_server_handlers_read.go
+++ b/weed/server/filer_server_handlers_read.go
@@ -52,6 +52,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 	if r.Method == "HEAD" {
 		w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10))
 		w.Header().Set("Last-Modified", entry.Attr.Mtime.Format(http.TimeFormat))
+		setEtag(w, filer2.ETag(entry.Chunks))
 		return
 	}