gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/chrislusf/seaweedfs into gocdk

Browse source
This commit is contained in:
Jonathan Amsterdam 2019-03-20 16:02:38 -04:00
parent 8db82e2b75 977b30e992
commit e966033098
27 changed files with 331 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
weed/command/benchmark.go
View file

@ -36,6 +36,7 @@ type BenchmarkOptions struct {
read *bool
sequentialRead *bool
collection *string
replication *string
cpuprofile *string
maxCpu *int
grpcDialOption grpc.DialOption
@ -61,6 +62,7 @@ func init() {
b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
sharedBytes = make([]byte, 1024)
@ -228,8 +230,9 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
MimeType: "image/bench", // prevent gzip benchmark content
}
ar := &operation.VolumeAssignRequest{
Count: 1,
Collection: *b.collection,
Count: 1,
Collection: *b.collection,
Replication: *b.replication,
}
if assignResult, err := operation.Assign(masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection

7
weed/command/mount.go
View file

@ -8,7 +8,6 @@ import (
type MountOptions struct {
filer *string
filerGrpcPort *int
filerMountRootPath *string
dir *string
dirListingLimit *int
@ -29,7 +28,6 @@ var (
func init() {
cmdMount.Run = runMount // break init cycle
mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
mountOptions.filerGrpcPort = cmdMount.Flag.Int("filer.grpc.port", 0, "filer grpc server listen port, default to http port + 10000")
mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 100000, "limit directory listing size")
@ -61,7 +59,7 @@ var cmdMount = &Command{
`,
}
func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress string, err error) {
func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
hostnameAndPort := strings.Split(filer, ":")
if len(hostnameAndPort) != 2 {
return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort)
@ -73,9 +71,6 @@ func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress
}
filerGrpcPort := int(filerPort) + 10000
if optionalGrpcPort != 0 {
filerGrpcPort = optionalGrpcPort
}
return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
}

2
weed/command/mount_std.go
View file

@ -87,7 +87,7 @@ func runMount(cmd *Command, args []string) bool {
c.Close()
})
filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer, *mountOptions.filerGrpcPort)
filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer)
if err != nil {
glog.Fatal(err)
return false

2
weed/command/s3.go
View file

@ -51,7 +51,7 @@ func runS3(cmd *Command, args []string) bool {
weed_server.LoadConfiguration("security", false)
filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer)
if err != nil {
glog.Fatal(err)
return false

2
weed/operation/grpc_client.go
View file

@ -42,7 +42,7 @@ func withMasterServerClient(masterServer string, grpcDialOption grpc.DialOption,
ctx := context.Background()
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer, 0)
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer)
if parseErr != nil {
return fmt.Errorf("failed to parse master grpc %v", masterServer)
}

1
weed/pb/master.proto
View file

@ -56,6 +56,7 @@ message VolumeInformationMessage {
uint32 replica_placement = 8;
uint32 version = 9;
uint32 ttl = 10;
uint32 compact_revision = 11;
}
message Empty {

187
weed/pb/master_pb/master.pb.go
View file

@ -190,6 +190,7 @@ type VolumeInformationMessage struct {
ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"`
Version uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"`
Ttl uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
CompactRevision uint32 `protobuf:"varint,11,opt,name=compact_revision,json=compactRevision" json:"compact_revision,omitempty"`
}
func (m *VolumeInformationMessage) Reset() { *m = VolumeInformationMessage{} }
@ -267,6 +268,13 @@ func (m *VolumeInformationMessage) GetTtl() uint32 {
return 0
}
func (m *VolumeInformationMessage) GetCompactRevision() uint32 {
if m != nil {
return m.CompactRevision
}
return 0
}
type Empty struct {
}
@ -1428,93 +1436,94 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("master.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1394 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0x4b, 0x6f, 0xdb, 0xc6,
0x13, 0x37, 0xa9, 0x87, 0xc5, 0xd1, 0xc3, 0xd2, 0xda, 0x49, 0x18, 0xe5, 0x9f, 0x44, 0x61, 0x2e,
0xfa, 0xbf, 0x8c, 0xd4, 0x3d, 0xf4, 0xd0, 0x16, 0x41, 0xe2, 0x38, 0x68, 0x10, 0xb7, 0x49, 0xe8,
0x24, 0x05, 0x0a, 0x14, 0xea, 0x9a, 0x1c, 0xbb, 0x84, 0x29, 0x92, 0x25, 0x57, 0x8e, 0x95, 0x4b,
0x2f, 0x3d, 0x16, 0xed, 0xa1, 0xdf, 0xa7, 0x97, 0xf6, 0xd6, 0x8f, 0xd2, 0x5b, 0xef, 0x05, 0x8a,
0x7d, 0x90, 0x5a, 0x52, 0xb2, 0x13, 0x14, 0xe8, 0x21, 0xb7, 0xdd, 0x99, 0xd9, 0xdd, 0xe1, 0xef,
0x37, 0x2f, 0x09, 0x3a, 0x53, 0x9a, 0x31, 0x4c, 0xb7, 0x93, 0x34, 0x66, 0x31, 0xb1, 0xe4, 0x6e,
0x92, 0x1c, 0x3a, 0xbf, 0x9b, 0x60, 0x7d, 0x82, 0x34, 0x65, 0x87, 0x48, 0x19, 0xe9, 0x81, 0x19,
0x24, 0xb6, 0x31, 0x32, 0xc6, 0x96, 0x6b, 0x06, 0x09, 0x21, 0x50, 0x4f, 0xe2, 0x94, 0xd9, 0xe6,
0xc8, 0x18, 0x77, 0x5d, 0xb1, 0x26, 0xd7, 0x01, 0x92, 0xd9, 0x61, 0x18, 0x78, 0x93, 0x59, 0x1a,
0xda, 0x35, 0x61, 0x6b, 0x49, 0xc9, 0x8b, 0x34, 0x24, 0x63, 0xe8, 0x4f, 0xe9, 0xd9, 0xe4, 0x34,
0x0e, 0x67, 0x53, 0x9c, 0x78, 0xf1, 0x2c, 0x62, 0x76, 0x5d, 0x1c, 0xef, 0x4d, 0xe9, 0xd9, 0x4b,
0x21, 0xde, 0xe5, 0x52, 0x32, 0xe2, 0x5e, 0x9d, 0x4d, 0x8e, 0x82, 0x10, 0x27, 0x27, 0x38, 0xb7,
0x1b, 0x23, 0x63, 0x5c, 0x77, 0x61, 0x4a, 0xcf, 0x1e, 0x06, 0x21, 0x3e, 0xc6, 0x39, 0xb9, 0x09,
0x6d, 0x9f, 0x32, 0x3a, 0xf1, 0x30, 0x62, 0x98, 0xda, 0x4d, 0xf1, 0x16, 0x70, 0xd1, 0xae, 0x90,
0x70, 0xff, 0x52, 0xea, 0x9d, 0xd8, 0xeb, 0x42, 0x23, 0xd6, 0xdc, 0x3f, 0xea, 0x4f, 0x83, 0x68,
0x22, 0x3c, 0x6f, 0x89, 0xa7, 0x2d, 0x21, 0x79, 0xca, 0xdd, 0xff, 0x18, 0xd6, 0xa5, 0x6f, 0x99,
0x6d, 0x8d, 0x6a, 0xe3, 0xf6, 0xce, 0xed, 0xed, 0x02, 0x8d, 0x6d, 0xe9, 0xde, 0xa3, 0xe8, 0x28,
0x4e, 0xa7, 0x94, 0x05, 0x71, 0xf4, 0x29, 0x66, 0x19, 0x3d, 0x46, 0x37, 0x3f, 0x43, 0xae, 0x42,
0x2b, 0xc2, 0x57, 0x93, 0xd3, 0xc0, 0xcf, 0x6c, 0x18, 0xd5, 0xc6, 0x5d, 0x77, 0x3d, 0xc2, 0x57,
0x2f, 0x03, 0x3f, 0x23, 0xb7, 0xa0, 0xe3, 0x63, 0x88, 0x0c, 0x7d, 0xa9, 0x6e, 0x0b, 0x75, 0x5b,
0xc9, 0xb8, 0x89, 0xf3, 0x02, 0x06, 0x05, 0xd8, 0x2e, 0x66, 0x49, 0x1c, 0x65, 0x48, 0xc6, 0xb0,
0x21, 0x6f, 0x3f, 0x08, 0x5e, 0xe3, 0x7e, 0x30, 0x0d, 0x98, 0x60, 0xa0, 0xee, 0x56, 0xc5, 0xe4,
0x32, 0x34, 0x43, 0xa4, 0x3e, 0xa6, 0x0a, 0x76, 0xb5, 0x73, 0x7e, 0x35, 0xc1, 0x3e, 0xcf, 0x75,
0xc1, 0xa9, 0x2f, 0x6e, 0xec, 0xba, 0x66, 0xe0, 0x73, 0xcc, 0xb2, 0xe0, 0x35, 0x0a, 0x4e, 0xeb,
0xae, 0x58, 0x93, 0x1b, 0x00, 0x5e, 0x1c, 0x86, 0xe8, 0xf1, 0x83, 0xea, 0x72, 0x4d, 0xc2, 0x31,
0x15, 0x34, 0x2d, 0xe8, 0xac, 0xbb, 0x16, 0x97, 0x48, 0x26, 0x8b, 0x2f, 0x57, 0x06, 0x92, 0x49,
0xf5, 0xe5, 0xd2, 0xe4, 0x7f, 0x40, 0x72, 0x70, 0x0e, 0xe7, 0x85, 0x61, 0x53, 0x18, 0xf6, 0x95,
0xe6, 0xfe, 0x3c, 0xb7, 0xbe, 0x06, 0x56, 0x8a, 0xd4, 0x9f, 0xc4, 0x51, 0x38, 0x17, 0xe4, 0xb6,
0xdc, 0x16, 0x17, 0x3c, 0x89, 0xc2, 0x39, 0xf9, 0x2f, 0x0c, 0x52, 0x4c, 0xc2, 0xc0, 0xa3, 0x93,
0x24, 0xa4, 0x1e, 0x4e, 0x31, 0xca, 0x79, 0xee, 0x2b, 0xc5, 0xd3, 0x5c, 0x4e, 0x6c, 0x58, 0x3f,
0xc5, 0x34, 0xe3, 0x9f, 0x65, 0x09, 0x93, 0x7c, 0x4b, 0xfa, 0x50, 0x63, 0x2c, 0xb4, 0x41, 0x48,
0xf9, 0xd2, 0x59, 0x87, 0xc6, 0xde, 0x34, 0x61, 0x73, 0xe7, 0x67, 0x03, 0x36, 0x0e, 0x66, 0x09,
0xa6, 0xf7, 0xc3, 0xd8, 0x3b, 0xd9, 0x3b, 0x63, 0x29, 0x25, 0x4f, 0xa0, 0x87, 0x29, 0xcd, 0x66,
0x29, 0xf7, 0xdd, 0x0f, 0xa2, 0x63, 0x01, 0x69, 0x7b, 0x67, 0xac, 0x85, 0x4f, 0xe5, 0xcc, 0xf6,
0x9e, 0x3c, 0xb0, 0x2b, 0xec, 0xdd, 0x2e, 0xea, 0xdb, 0xe1, 0x17, 0xd0, 0x2d, 0xe9, 0x39, 0x31,
0x3c, 0xb4, 0x15, 0x55, 0x62, 0xcd, 0x19, 0x4f, 0x68, 0x1a, 0xb0, 0xb9, 0x4a, 0x41, 0xb5, 0xe3,
0x84, 0xa8, 0x0c, 0xe3, 0x91, 0x56, 0x13, 0x91, 0x66, 0x49, 0xc9, 0x23, 0x3f, 0x73, 0xfe, 0x0d,
0x9b, 0xbb, 0x61, 0x80, 0x11, 0xdb, 0x0f, 0x32, 0x86, 0x91, 0x8b, 0xdf, 0xcc, 0x30, 0x63, 0xfc,
0x85, 0x88, 0x4e, 0x51, 0x25, 0xb8, 0x58, 0x3b, 0xdf, 0x42, 0x4f, 0x86, 0xce, 0x7e, 0xec, 0x89,
0xb8, 0xe1, 0xc0, 0xf0, 0xcc, 0x96, 0x46, 0x7c, 0x59, 0x49, 0x79, 0xb3, 0x9a, 0xf2, 0x7a, 0x4e,
0xd4, 0x2e, 0xce, 0x89, 0xfa, 0x72, 0x4e, 0x3c, 0x87, 0xcd, 0xfd, 0x38, 0x3e, 0x99, 0x25, 0xd2,
0x8d, 0xdc, 0xd7, 0xf2, 0x17, 0x1a, 0xa3, 0x1a, 0x7f, 0xb3, 0xf8, 0xc2, 0x4a, 0xc4, 0x9a, 0xd5,
0x88, 0x75, 0xfe, 0x30, 0x60, 0xab, 0x7c, 0xad, 0xca, 0xb6, 0xaf, 0x60, 0xb3, 0xb8, 0x77, 0x12,
0xaa, 0x6f, 0x96, 0x0f, 0xb4, 0x77, 0xee, 0x68, 0x64, 0xae, 0x3a, 0x9d, 0x17, 0x08, 0x3f, 0x07,
0xcb, 0x1d, 0x9c, 0x56, 0x24, 0xd9, 0xf0, 0x0c, 0xfa, 0x55, 0x33, 0x1e, 0xd0, 0xc5, 0xab, 0x0a,
0xd9, 0x56, 0x7e, 0x92, 0xbc, 0x07, 0xd6, 0xc2, 0x11, 0x53, 0x38, 0xb2, 0x59, 0x72, 0x44, 0xbd,
0xb5, 0xb0, 0x22, 0x5b, 0xd0, 0xc0, 0x34, 0x8d, 0xf3, 0x42, 0x20, 0x37, 0xce, 0x87, 0xd0, 0xfa,
0xdb, 0x2c, 0x3a, 0xbf, 0x19, 0xd0, 0xbd, 0x97, 0x65, 0xc1, 0x71, 0x11, 0x2e, 0x5b, 0xd0, 0x90,
0x69, 0x2a, 0xcb, 0x91, 0xdc, 0x90, 0x11, 0xb4, 0x55, 0x96, 0x69, 0xd0, 0xeb, 0xa2, 0x37, 0x56,
0x13, 0x95, 0x79, 0x75, 0xe9, 0x1a, 0x63, 0x61, 0xb5, 0xd0, 0x37, 0xce, 0x2d, 0xf4, 0x4d, 0xad,
0xd0, 0x5f, 0x03, 0x4b, 0x1c, 0x8a, 0x62, 0x1f, 0x55, 0x07, 0x68, 0x71, 0xc1, 0x67, 0xb1, 0x8f,
0xce, 0x4f, 0x06, 0xf4, 0xf2, 0xaf, 0x51, 0xcc, 0xf7, 0xa1, 0x76, 0x54, 0xa0, 0xcf, 0x97, 0x39,
0x46, 0xe6, 0x79, 0x18, 0x2d, 0x35, 0xb7, 0x02, 0x91, 0xba, 0x8e, 0x48, 0x41, 0x46, 0x43, 0x23,
0x83, 0xbb, 0x4c, 0x67, 0xec, 0xeb, 0xdc, 0x65, 0xbe, 0x76, 0x8e, 0x61, 0x70, 0xc0, 0x28, 0x0b,
0x32, 0x16, 0x78, 0x59, 0x0e, 0x73, 0x05, 0x50, 0xe3, 0x4d, 0x80, 0x9a, 0xe7, 0x01, 0x5a, 0x2b,
0x00, 0x75, 0x7e, 0x31, 0x80, 0xe8, 0x2f, 0x29, 0x08, 0xfe, 0x81, 0xa7, 0x38, 0x64, 0x2c, 0x66,
0x34, 0x9c, 0x88, 0xae, 0xa2, 0x7a, 0x83, 0x90, 0xf0, 0xc6, 0xc5, 0x59, 0x9a, 0x65, 0xe8, 0x4b,
0xad, 0x6c, 0x0c, 0x2d, 0x2e, 0x10, 0xca, 0x72, 0x5f, 0x69, 0x56, 0xfa, 0x8a, 0x73, 0x0f, 0xda,
0x07, 0x2c, 0x4e, 0xe9, 0x31, 0x3e, 0x9f, 0x27, 0x6f, 0xe3, 0xbd, 0xf2, 0xce, 0x5c, 0x00, 0x31,
0x02, 0xd8, 0x5d, 0x78, 0xbf, 0xaa, 0x00, 0x5e, 0x81, 0x4b, 0x0b, 0x0b, 0x5e, 0x2f, 0x15, 0x2f,
0xce, 0x33, 0xb8, 0x5c, 0x55, 0x28, 0x18, 0x3f, 0x80, 0xf6, 0x02, 0x92, 0xbc, 0x76, 0x5c, 0xd2,
0x52, 0x76, 0x71, 0xce, 0xd5, 0x2d, 0x9d, 0xff, 0xc3, 0x95, 0x85, 0xea, 0x81, 0x28, 0x82, 0x17,
0xd5, 0xe6, 0x21, 0xd8, 0xcb, 0xe6, 0xd2, 0x07, 0xe7, 0x47, 0x13, 0x3a, 0x0f, 0x54, 0xb4, 0xf3,
0xae, 0xaf, 0xf5, 0x79, 0x4b, 0xf4, 0xf9, 0x5b, 0xd0, 0x29, 0x0d, 0x61, 0xb2, 0xdf, 0xb7, 0x4f,
0xb5, 0x09, 0x6c, 0xd5, 0xac, 0x56, 0x13, 0x66, 0xd5, 0x59, 0xed, 0x3f, 0x30, 0x38, 0x4a, 0x11,
0x97, 0xc7, 0xba, 0xba, 0xbb, 0xc1, 0x15, 0xba, 0xed, 0x36, 0x6c, 0x52, 0x8f, 0x05, 0xa7, 0x15,
0x6b, 0xc9, 0xfd, 0x40, 0xaa, 0x74, 0xfb, 0x87, 0x85, 0xa3, 0x41, 0x74, 0x14, 0x67, 0x76, 0xf3,
0xed, 0xc7, 0x32, 0xf5, 0x35, 0x5c, 0x93, 0x39, 0xdf, 0x99, 0xd0, 0x72, 0xa9, 0x77, 0xf2, 0x6e,
0xa3, 0x71, 0x17, 0x36, 0x8a, 0xaa, 0x56, 0x02, 0xe4, 0x8a, 0x06, 0x88, 0x4e, 0xbc, 0xdb, 0xf5,
0xb5, 0x5d, 0xe6, 0xfc, 0x69, 0x40, 0xef, 0x41, 0x51, 0x39, 0xdf, 0x6d, 0x30, 0x76, 0x00, 0x78,
0xa9, 0x2f, 0xe1, 0xa0, 0xb7, 0xc6, 0x9c, 0x6e, 0xd7, 0x4a, 0xd5, 0x2a, 0x73, 0x7e, 0x30, 0xa1,
0xf3, 0x3c, 0x4e, 0xe2, 0x30, 0x3e, 0x9e, 0xbf, 0xdb, 0x5f, 0xbf, 0x07, 0x03, 0xad, 0x2b, 0x96,
0x40, 0xb8, 0x5a, 0x09, 0x86, 0x05, 0xd9, 0xee, 0x86, 0x5f, 0xda, 0x67, 0xce, 0x26, 0x0c, 0xd4,
0x84, 0xa7, 0x15, 0x37, 0x17, 0x88, 0x2e, 0x54, 0x85, 0xed, 0x23, 0xe8, 0x32, 0x05, 0x9d, 0x78,
0x4e, 0xcd, 0xb8, 0x7a, 0xe8, 0xe9, 0xd0, 0xba, 0x1d, 0xa6, 0xed, 0x76, 0xbe, 0x6f, 0xc0, 0xfa,
0x01, 0xd2, 0x57, 0x88, 0x3e, 0x79, 0x04, 0xdd, 0x03, 0x8c, 0xfc, 0xc5, 0x4f, 0xcb, 0x2d, 0xed,
0x8e, 0x42, 0x3a, 0xfc, 0xd7, 0x2a, 0x69, 0x51, 0xe3, 0xd6, 0xc6, 0xc6, 0x1d, 0x83, 0x3c, 0x85,
0xee, 0x63, 0xc4, 0x64, 0x37, 0x8e, 0x22, 0xf4, 0x18, 0xfa, 0xe4, 0x86, 0x5e, 0x69, 0x97, 0xc7,
0xdc, 0xe1, 0xd5, 0xa5, 0xd2, 0x91, 0x4f, 0x45, 0xea, 0xc6, 0x67, 0xd0, 0xd1, 0xa7, 0xbb, 0xd2,
0x85, 0x2b, 0x66, 0xd1, 0xe1, 0xcd, 0x37, 0x8c, 0x85, 0xce, 0x1a, 0xb9, 0x0b, 0x4d, 0x39, 0x6e,
0x10, 0x5b, 0x33, 0x2e, 0xcd, 0x53, 0x25, 0xbf, 0xca, 0xb3, 0x89, 0xb3, 0x46, 0x1e, 0x03, 0x2c,
0x1a, 0x36, 0xd1, 0x71, 0x59, 0x9a, 0x18, 0x86, 0xd7, 0xcf, 0xd1, 0x16, 0x97, 0x7d, 0x0e, 0xbd,
0x72, 0xeb, 0x22, 0xa3, 0x95, 0xdd, 0x49, 0x8b, 0x88, 0xe1, 0xad, 0x0b, 0x2c, 0x8a, 0x8b, 0xbf,
0x84, 0x7e, 0xb5, 0x23, 0x11, 0x67, 0xe5, 0xc1, 0x52, 0x77, 0x1b, 0xde, 0xbe, 0xd0, 0x46, 0x07,
0x61, 0x11, 0x95, 0x25, 0x10, 0x96, 0x22, 0xb8, 0x04, 0xc2, 0x72, 0x28, 0x3b, 0x6b, 0x87, 0x4d,
0xf1, 0x67, 0xc7, 0xfb, 0x7f, 0x05, 0x00, 0x00, 0xff, 0xff, 0xbf, 0x50, 0x28, 0x66, 0xfc, 0x10,
0x00, 0x00,
// 1416 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6f, 0xdc, 0x44,
0x14, 0x8f, 0xbd, 0x1f, 0x59, 0xbf, 0xfd, 0xc8, 0xee, 0x24, 0x6d, 0xdd, 0x2d, 0x6d, 0xb7, 0xee,
0x65, 0xcb, 0x47, 0x54, 0xc2, 0x81, 0x03, 0xa0, 0xaa, 0x4d, 0x53, 0x51, 0x35, 0xd0, 0xd6, 0x69,
0x8b, 0x84, 0x84, 0xcc, 0xc4, 0x7e, 0x09, 0x56, 0xbc, 0xb6, 0xb1, 0x67, 0xb7, 0xd9, 0x5e, 0xb8,
0x70, 0x44, 0x70, 0xe0, 0xff, 0xe1, 0xc2, 0x91, 0x3f, 0x85, 0x03, 0x12, 0x77, 0x24, 0x34, 0xe3,
0xb1, 0x77, 0xec, 0xdd, 0xa4, 0x15, 0x12, 0x87, 0xde, 0x3c, 0xbf, 0xf7, 0x66, 0xe6, 0xcd, 0xef,
0x7d, 0xee, 0x42, 0x67, 0x42, 0x53, 0x86, 0xc9, 0x76, 0x9c, 0x44, 0x2c, 0x22, 0x46, 0xb6, 0x72,
0xe2, 0x43, 0xeb, 0x4f, 0x1d, 0x8c, 0xcf, 0x91, 0x26, 0xec, 0x10, 0x29, 0x23, 0x3d, 0xd0, 0xfd,
0xd8, 0xd4, 0x46, 0xda, 0xd8, 0xb0, 0x75, 0x3f, 0x26, 0x04, 0xea, 0x71, 0x94, 0x30, 0x53, 0x1f,
0x69, 0xe3, 0xae, 0x2d, 0xbe, 0xc9, 0x55, 0x80, 0x78, 0x7a, 0x18, 0xf8, 0xae, 0x33, 0x4d, 0x02,
0xb3, 0x26, 0x74, 0x8d, 0x0c, 0x79, 0x9e, 0x04, 0x64, 0x0c, 0xfd, 0x09, 0x3d, 0x75, 0x66, 0x51,
0x30, 0x9d, 0xa0, 0xe3, 0x46, 0xd3, 0x90, 0x99, 0x75, 0xb1, 0xbd, 0x37, 0xa1, 0xa7, 0x2f, 0x04,
0xbc, 0xcb, 0x51, 0x32, 0xe2, 0x56, 0x9d, 0x3a, 0x47, 0x7e, 0x80, 0xce, 0x09, 0xce, 0xcd, 0xc6,
0x48, 0x1b, 0xd7, 0x6d, 0x98, 0xd0, 0xd3, 0x07, 0x7e, 0x80, 0x8f, 0x70, 0x4e, 0xae, 0x43, 0xdb,
0xa3, 0x8c, 0x3a, 0x2e, 0x86, 0x0c, 0x13, 0xb3, 0x29, 0xee, 0x02, 0x0e, 0xed, 0x0a, 0x84, 0xdb,
0x97, 0x50, 0xf7, 0xc4, 0x5c, 0x17, 0x12, 0xf1, 0xcd, 0xed, 0xa3, 0xde, 0xc4, 0x0f, 0x1d, 0x61,
0x79, 0x4b, 0x5c, 0x6d, 0x08, 0xe4, 0x09, 0x37, 0xff, 0x33, 0x58, 0xcf, 0x6c, 0x4b, 0x4d, 0x63,
0x54, 0x1b, 0xb7, 0x77, 0x6e, 0x6e, 0x17, 0x6c, 0x6c, 0x67, 0xe6, 0x3d, 0x0c, 0x8f, 0xa2, 0x64,
0x42, 0x99, 0x1f, 0x85, 0x5f, 0x60, 0x9a, 0xd2, 0x63, 0xb4, 0xf3, 0x3d, 0xe4, 0x32, 0xb4, 0x42,
0x7c, 0xe9, 0xcc, 0x7c, 0x2f, 0x35, 0x61, 0x54, 0x1b, 0x77, 0xed, 0xf5, 0x10, 0x5f, 0xbe, 0xf0,
0xbd, 0x94, 0xdc, 0x80, 0x8e, 0x87, 0x01, 0x32, 0xf4, 0x32, 0x71, 0x5b, 0x88, 0xdb, 0x12, 0xe3,
0x2a, 0xd6, 0x73, 0x18, 0x14, 0x64, 0xdb, 0x98, 0xc6, 0x51, 0x98, 0x22, 0x19, 0xc3, 0x46, 0x76,
0xfa, 0x81, 0xff, 0x0a, 0xf7, 0xfd, 0x89, 0xcf, 0x84, 0x07, 0xea, 0x76, 0x15, 0x26, 0x17, 0xa1,
0x19, 0x20, 0xf5, 0x30, 0x91, 0xb4, 0xcb, 0x95, 0xf5, 0x97, 0x0e, 0xe6, 0x59, 0xa6, 0x0b, 0x9f,
0x7a, 0xe2, 0xc4, 0xae, 0xad, 0xfb, 0x1e, 0xe7, 0x2c, 0xf5, 0x5f, 0xa1, 0xf0, 0x69, 0xdd, 0x16,
0xdf, 0xe4, 0x1a, 0x80, 0x1b, 0x05, 0x01, 0xba, 0x7c, 0xa3, 0x3c, 0x5c, 0x41, 0x38, 0xa7, 0xc2,
0x4d, 0x0b, 0x77, 0xd6, 0x6d, 0x83, 0x23, 0x99, 0x27, 0x8b, 0x97, 0x4b, 0x85, 0xcc, 0x93, 0xf2,
0xe5, 0x99, 0xca, 0xfb, 0x40, 0x72, 0x72, 0x0e, 0xe7, 0x85, 0x62, 0x53, 0x28, 0xf6, 0xa5, 0xe4,
0xde, 0x3c, 0xd7, 0xbe, 0x02, 0x46, 0x82, 0xd4, 0x73, 0xa2, 0x30, 0x98, 0x0b, 0xe7, 0xb6, 0xec,
0x16, 0x07, 0x1e, 0x87, 0xc1, 0x9c, 0xbc, 0x07, 0x83, 0x04, 0xe3, 0xc0, 0x77, 0xa9, 0x13, 0x07,
0xd4, 0xc5, 0x09, 0x86, 0xb9, 0x9f, 0xfb, 0x52, 0xf0, 0x24, 0xc7, 0x89, 0x09, 0xeb, 0x33, 0x4c,
0x52, 0xfe, 0x2c, 0x43, 0xa8, 0xe4, 0x4b, 0xd2, 0x87, 0x1a, 0x63, 0x81, 0x09, 0x02, 0xe5, 0x9f,
0xe4, 0x16, 0xf4, 0xdd, 0x68, 0x12, 0x53, 0x97, 0x39, 0x09, 0xce, 0x7c, 0xb1, 0xa9, 0x2d, 0xc4,
0x1b, 0x12, 0xb7, 0x25, 0x6c, 0xad, 0x43, 0x63, 0x6f, 0x12, 0xb3, 0xb9, 0xf5, 0x9b, 0x06, 0x1b,
0x07, 0xd3, 0x18, 0x93, 0x7b, 0x41, 0xe4, 0x9e, 0xec, 0x9d, 0xb2, 0x84, 0x92, 0xc7, 0xd0, 0xc3,
0x84, 0xa6, 0xd3, 0x84, 0x3f, 0xd3, 0xf3, 0xc3, 0x63, 0xc1, 0x7e, 0x7b, 0x67, 0xac, 0x44, 0x5a,
0x65, 0xcf, 0xf6, 0x5e, 0xb6, 0x61, 0x57, 0xe8, 0xdb, 0x5d, 0x54, 0x97, 0xc3, 0xaf, 0xa1, 0x5b,
0x92, 0x73, 0x1f, 0xf2, 0x2c, 0x90, 0x5e, 0x15, 0xdf, 0x3c, 0x38, 0x62, 0x9a, 0xf8, 0x6c, 0x2e,
0xb3, 0x55, 0xae, 0xb8, 0xef, 0x64, 0x32, 0xf2, 0xa0, 0xac, 0x89, 0xa0, 0x34, 0x32, 0xe4, 0xa1,
0x97, 0x5a, 0xb7, 0x60, 0x73, 0x37, 0xf0, 0x31, 0x64, 0xfb, 0x7e, 0xca, 0x30, 0xb4, 0xf1, 0xfb,
0x29, 0xa6, 0x8c, 0xdf, 0x10, 0xd2, 0x09, 0xca, 0x5a, 0x20, 0xbe, 0xad, 0x1f, 0xa0, 0x97, 0x45,
0xd9, 0x7e, 0xe4, 0x8a, 0x10, 0xe3, 0x1c, 0xf2, 0x22, 0x90, 0x29, 0xf1, 0xcf, 0x4a, 0x75, 0xd0,
0xab, 0xd5, 0x41, 0x4d, 0x9f, 0xda, 0xf9, 0xe9, 0x53, 0x5f, 0x4e, 0x9f, 0x67, 0xb0, 0xb9, 0x1f,
0x45, 0x27, 0xd3, 0x38, 0x33, 0x23, 0xb7, 0xb5, 0xfc, 0x42, 0x6d, 0x54, 0xe3, 0x77, 0x16, 0x2f,
0xac, 0x04, 0xb7, 0x5e, 0x0d, 0x6e, 0xeb, 0x6f, 0x0d, 0xb6, 0xca, 0xc7, 0xca, 0xc4, 0xfc, 0x16,
0x36, 0x8b, 0x73, 0x9d, 0x40, 0xbe, 0x39, 0xbb, 0xa0, 0xbd, 0x73, 0x5b, 0x71, 0xe6, 0xaa, 0xdd,
0x79, 0x2d, 0xf1, 0x72, 0xb2, 0xec, 0xc1, 0xac, 0x82, 0xa4, 0xc3, 0x53, 0xe8, 0x57, 0xd5, 0x78,
0xec, 0x17, 0xb7, 0x4a, 0x66, 0x5b, 0xf9, 0x4e, 0xf2, 0x21, 0x18, 0x0b, 0x43, 0x74, 0x61, 0xc8,
0x66, 0xc9, 0x10, 0x79, 0xd7, 0x42, 0x8b, 0x6c, 0x41, 0x03, 0x93, 0x24, 0xca, 0x6b, 0x46, 0xb6,
0xb0, 0x3e, 0x81, 0xd6, 0x7f, 0xf6, 0xa2, 0xf5, 0x87, 0x06, 0xdd, 0xbb, 0x69, 0xea, 0x1f, 0x17,
0xe1, 0xb2, 0x05, 0x8d, 0x2c, 0xa3, 0xb3, 0xca, 0x95, 0x2d, 0xc8, 0x08, 0xda, 0x32, 0x21, 0x15,
0xea, 0x55, 0xe8, 0xb5, 0x85, 0x47, 0x26, 0x69, 0x3d, 0x33, 0x8d, 0x27, 0x69, 0xa5, 0x27, 0x34,
0xce, 0xec, 0x09, 0x4d, 0xa5, 0x27, 0x5c, 0x01, 0x43, 0x6c, 0x0a, 0x23, 0x0f, 0x65, 0xb3, 0x68,
0x71, 0xe0, 0xcb, 0xc8, 0x43, 0xeb, 0x57, 0x0d, 0x7a, 0xf9, 0x6b, 0xa4, 0xe7, 0xfb, 0x50, 0x3b,
0x2a, 0xd8, 0xe7, 0x9f, 0x39, 0x47, 0xfa, 0x59, 0x1c, 0x2d, 0xf5, 0xc1, 0x82, 0x91, 0xba, 0xca,
0x48, 0xe1, 0x8c, 0x86, 0xe2, 0x0c, 0x6e, 0x32, 0x9d, 0xb2, 0xef, 0x72, 0x93, 0xf9, 0xb7, 0x75,
0x0c, 0x83, 0x03, 0x46, 0x99, 0x9f, 0x32, 0xdf, 0x4d, 0x73, 0x9a, 0x2b, 0x84, 0x6a, 0xaf, 0x23,
0x54, 0x3f, 0x8b, 0xd0, 0x5a, 0x41, 0xa8, 0xf5, 0xbb, 0x06, 0x44, 0xbd, 0x49, 0x52, 0xf0, 0x3f,
0x5c, 0xc5, 0x29, 0x63, 0x11, 0xa3, 0x81, 0x23, 0x1a, 0x90, 0x6c, 0x23, 0x02, 0xe1, 0x3d, 0x8e,
0x7b, 0x69, 0x9a, 0xa2, 0x97, 0x49, 0xb3, 0x1e, 0xd2, 0xe2, 0x80, 0x10, 0x96, 0x5b, 0x50, 0xb3,
0xd2, 0x82, 0xac, 0xbb, 0xd0, 0x3e, 0x60, 0x51, 0x42, 0x8f, 0xf1, 0xd9, 0x3c, 0x7e, 0x13, 0xeb,
0xa5, 0x75, 0xfa, 0x82, 0x88, 0x11, 0xc0, 0xee, 0xc2, 0xfa, 0x55, 0x05, 0xf0, 0x12, 0x5c, 0x58,
0x68, 0xf0, 0x7a, 0x29, 0xfd, 0x62, 0x3d, 0x85, 0x8b, 0x55, 0x81, 0xa4, 0xf1, 0x63, 0x68, 0x2f,
0x28, 0xc9, 0x6b, 0xc7, 0x05, 0x25, 0x65, 0x17, 0xfb, 0x6c, 0x55, 0xd3, 0xfa, 0x00, 0x2e, 0x2d,
0x44, 0xf7, 0x45, 0x11, 0x3c, 0xaf, 0x36, 0x0f, 0xc1, 0x5c, 0x56, 0xcf, 0x6c, 0xb0, 0x7e, 0xd1,
0xa1, 0x73, 0x5f, 0x46, 0x3b, 0x1f, 0x10, 0x94, 0x91, 0xc0, 0x10, 0x23, 0xc1, 0x0d, 0xe8, 0x94,
0xe6, 0xb5, 0x6c, 0x34, 0x68, 0xcf, 0x94, 0x61, 0x6d, 0xd5, 0x58, 0x57, 0x13, 0x6a, 0xd5, 0xb1,
0xee, 0x5d, 0x18, 0x1c, 0x25, 0x88, 0xcb, 0x13, 0x60, 0xdd, 0xde, 0xe0, 0x02, 0x55, 0x77, 0x1b,
0x36, 0xa9, 0xcb, 0xfc, 0x59, 0x45, 0x3b, 0xf3, 0xfd, 0x20, 0x13, 0xa9, 0xfa, 0x0f, 0x0a, 0x43,
0xfd, 0xf0, 0x28, 0x4a, 0xcd, 0xe6, 0x9b, 0x4f, 0x70, 0xf2, 0x35, 0x5c, 0x92, 0x5a, 0x3f, 0xea,
0xd0, 0xb2, 0xa9, 0x7b, 0xf2, 0x76, 0xb3, 0x71, 0x07, 0x36, 0x8a, 0xaa, 0x56, 0x22, 0xe4, 0x92,
0x42, 0x88, 0xea, 0x78, 0xbb, 0xeb, 0x29, 0xab, 0xd4, 0xfa, 0x47, 0x83, 0xde, 0xfd, 0xa2, 0x72,
0xbe, 0xdd, 0x64, 0xec, 0x00, 0xf0, 0x52, 0x5f, 0xe2, 0x41, 0x6d, 0x8d, 0xb9, 0xbb, 0x6d, 0x23,
0x91, 0x5f, 0xa9, 0xf5, 0xb3, 0x0e, 0x9d, 0x67, 0x51, 0x1c, 0x05, 0xd1, 0xf1, 0xfc, 0xed, 0x7e,
0xfd, 0x1e, 0x0c, 0x94, 0xae, 0x58, 0x22, 0xe1, 0x72, 0x25, 0x18, 0x16, 0xce, 0xb6, 0x37, 0xbc,
0xd2, 0x3a, 0xb5, 0x36, 0x61, 0x20, 0x27, 0x3c, 0xa5, 0xb8, 0xd9, 0x40, 0x54, 0x50, 0x16, 0xb6,
0x4f, 0xa1, 0xcb, 0x24, 0x75, 0xe2, 0x3a, 0x39, 0xe3, 0xaa, 0xa1, 0xa7, 0x52, 0x6b, 0x77, 0x98,
0xb2, 0xda, 0xf9, 0xa9, 0x01, 0xeb, 0x07, 0x48, 0x5f, 0x22, 0x7a, 0xe4, 0x21, 0x74, 0x0f, 0x30,
0xf4, 0x16, 0xbf, 0x42, 0xb7, 0x94, 0x33, 0x0a, 0x74, 0xf8, 0xce, 0x2a, 0xb4, 0xa8, 0x71, 0x6b,
0x63, 0xed, 0xb6, 0x46, 0x9e, 0x40, 0xf7, 0x11, 0x62, 0xbc, 0x1b, 0x85, 0x21, 0xba, 0x0c, 0x3d,
0x72, 0x4d, 0xad, 0xb4, 0xcb, 0x63, 0xee, 0xf0, 0xf2, 0x52, 0xe9, 0xc8, 0xa7, 0x22, 0x79, 0xe2,
0x53, 0xe8, 0xa8, 0xd3, 0x5d, 0xe9, 0xc0, 0x15, 0xb3, 0xe8, 0xf0, 0xfa, 0x6b, 0xc6, 0x42, 0x6b,
0x8d, 0xdc, 0x81, 0x66, 0x36, 0x6e, 0x10, 0x53, 0x51, 0x2e, 0xcd, 0x53, 0x25, 0xbb, 0xca, 0xb3,
0x89, 0xb5, 0x46, 0x1e, 0x01, 0x2c, 0x1a, 0x36, 0x51, 0x79, 0x59, 0x9a, 0x18, 0x86, 0x57, 0xcf,
0x90, 0x16, 0x87, 0x7d, 0x05, 0xbd, 0x72, 0xeb, 0x22, 0xa3, 0x95, 0xdd, 0x49, 0x89, 0x88, 0xe1,
0x8d, 0x73, 0x34, 0x8a, 0x83, 0xbf, 0x81, 0x7e, 0xb5, 0x23, 0x11, 0x6b, 0xe5, 0xc6, 0x52, 0x77,
0x1b, 0xde, 0x3c, 0x57, 0x47, 0x25, 0x61, 0x11, 0x95, 0x25, 0x12, 0x96, 0x22, 0xb8, 0x44, 0xc2,
0x72, 0x28, 0x5b, 0x6b, 0x87, 0x4d, 0xf1, 0xbf, 0xc8, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff,
0x88, 0xac, 0x76, 0x9c, 0x27, 0x11, 0x00, 0x00,
}

7
weed/server/filer_grpc_server.go
View file

@ -238,7 +238,12 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol
func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.DeleteCollectionRequest) (resp *filer_pb.DeleteCollectionResponse, err error) {
err = fs.filer.MasterClient.CollectionDelete(ctx, req.GetCollection())
err = fs.filer.MasterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
Name: req.GetCollection(),
})
return err
})
return &filer_pb.DeleteCollectionResponse{}, err
}

1
weed/server/filer_server_handlers_read.go
View file

@ -52,6 +52,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
if r.Method == "HEAD" {
w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10))
w.Header().Set("Last-Modified", entry.Attr.Mtime.Format(http.TimeFormat))
setEtag(w, filer2.ETag(entry.Chunks))
return
}

10
weed/server/filer_server_handlers_write_autochunk.go
View file

@ -19,7 +19,8 @@ import (
"github.com/chrislusf/seaweedfs/weed/util"
)
func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, replication string, collection string, dataCenter string) bool {
func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
replication string, collection string, dataCenter string) bool {
if r.Method != "POST" {
glog.V(4).Infoln("AutoChunking not supported for method", r.Method)
return false
@ -64,7 +65,8 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
return true
}
func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) {
func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) {
multipartReader, multipartReaderErr := r.MultipartReader()
if multipartReaderErr != nil {
@ -177,8 +179,8 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r
return
}
func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string, auth security.EncodedJwt) (err error) {
err = nil
func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request,
chunkBuf []byte, fileName string, contentType string, fileId string, auth security.EncodedJwt) (err error) {
ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf))
uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, nil, auth)

4
weed/server/raft_server.go
View file

@ -61,7 +61,7 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr st
s.raftServer.Start()
for _, peer := range s.peers {
s.raftServer.AddPeer(peer, util.ServerToGrpcAddress(peer, 19333))
s.raftServer.AddPeer(peer, util.ServerToGrpcAddress(peer))
}
s.GrpcServer = raft.NewGrpcServer(s.raftServer)
@ -72,7 +72,7 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []string, serverAddr st
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: util.ServerToGrpcAddress(s.serverAddr, 19333),
ConnectionString: util.ServerToGrpcAddress(s.serverAddr),
})
if err != nil {

2
weed/server/volume_grpc_client_to_master.go
View file

@ -31,7 +31,7 @@ func (vs *VolumeServer) heartbeat() {
if newLeader != "" {
master = newLeader
}
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master, 0)
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master)
if parseErr != nil {
glog.V(0).Infof("failed to parse master grpc %v", masterGrpcAddress)
continue

3
weed/server/volume_server_handlers_write.go
View file

@ -41,8 +41,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
}
ret := operation.UploadResult{}
_, errorStatus := topology.ReplicatedWrite(vs.GetMaster(),
vs.store, volumeId, needle, r)
_, errorStatus := topology.ReplicatedWrite(vs.GetMaster(), vs.store, volumeId, needle, r)
httpStatus := http.StatusCreated
if errorStatus != "" {
httpStatus = http.StatusInternalServerError

10
weed/shell/command_collection_list.go
View file

@ -3,6 +3,7 @@ package shell
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"io"
)
@ -21,9 +22,14 @@ func (c *commandCollectionList) Help() string {
return "# list all collections"
}
func (c *commandCollectionList) Do(args []string, commandEnv *commandEnv, writer io.Writer) error {
func (c *commandCollectionList) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
resp, err := commandEnv.masterClient.CollectionList(context.Background())
var resp *master_pb.CollectionListResponse
ctx := context.Background()
err = commandEnv.masterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
resp, err = client.CollectionList(ctx, &master_pb.CollectionListRequest{})
return err
})
if err != nil {
return err

137
weed/shell/command_fs_du.go Normal file
View file

@ -0,0 +1,137 @@
package shell
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
"io"
"net/url"
"strconv"
"strings"
)
func init() {
commands = append(commands, &commandFsDu{})
}
type commandFsDu struct {
}
func (c *commandFsDu) Name() string {
return "fs.du"
}
func (c *commandFsDu) Help() string {
return "http://<filer_server>:<port>/dir[/file] # show disk usage"
}
func (c *commandFsDu) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
filerServer, filerPort, path, err := parseFilerUrl(args[0])
if err != nil {
return err
}
dir, name := filer2.FullPath(path).DirAndName()
if strings.HasSuffix(path, "/") {
if path == "/"{
dir, name = "/", ""
}else{
dir, name = path[0 : len(path)-1], ""
}
}
ctx := context.Background()
return commandEnv.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
_, _, err = paginateDirectory(ctx, writer, client, dir, name, 1000)
return err
})
}
func paginateDirectory(ctx context.Context, writer io.Writer, client filer_pb.SeaweedFilerClient, dir, name string, paginateSize int) (blockCount uint64, byteCount uint64, err error) {
paginatedCount := -1
startFromFileName := ""
for paginatedCount == -1 || paginatedCount == paginateSize {
resp, listErr := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
Directory: dir,
Prefix: name,
StartFromFileName: startFromFileName,
InclusiveStartFrom: false,
Limit: uint32(paginateSize),
})
if listErr != nil {
err = listErr
return
}
paginatedCount = len(resp.Entries)
for _, entry := range resp.Entries {
if entry.IsDirectory {
subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
if dir == "/" {
subDir = "/" + entry.Name
}
numBlock, numByte, err := paginateDirectory(ctx, writer, client, subDir, "", paginateSize)
if err == nil {
blockCount += numBlock
byteCount += numByte
}
} else {
blockCount += uint64(len(entry.Chunks))
byteCount += filer2.TotalSize(entry.Chunks)
}
startFromFileName = entry.Name
if name != "" && !entry.IsDirectory {
fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s/%s\n", blockCount, byteCount, dir, name)
}
}
}
if name == "" {
fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir)
}
return
}
func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
if strings.HasPrefix(entryPath, "http") {
var u *url.URL
u, err = url.Parse(entryPath)
if err != nil {
return
}
filerServer = u.Hostname()
portString := u.Port()
if portString != "" {
filerPort, err = strconv.ParseInt(portString, 10, 32)
}
path = u.Path
} else {
err = fmt.Errorf("path should have full url http://<filer_server>:<port>/path/to/dirOrFile : %s", entryPath)
}
return
}
func (env *commandEnv) withFilerClient(ctx context.Context, filerServer string, filerPort int64, fn func(filer_pb.SeaweedFilerClient) error) error {
filerGrpcAddress := fmt.Sprintf("%s:%d", filerServer, filerPort+10000)
return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, filerGrpcAddress, env.option.GrpcDialOption)
}

11
weed/shell/command_volume_list.go
View file

@ -22,16 +22,19 @@ func (c *commandVolumeList) Help() string {
return "# list all volumes"
}
func (c *commandVolumeList) Do(args []string, commandEnv *commandEnv, writer io.Writer) error {
resp, err := commandEnv.masterClient.VolumeList(context.Background())
func (c *commandVolumeList) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
var resp *master_pb.VolumeListResponse
ctx := context.Background()
err = commandEnv.masterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
resp, err = client.VolumeList(ctx, &master_pb.VolumeListRequest{})
return err
})
if err != nil {
return err
}
writeTopologyInfo(writer, resp.TopologyInfo)
return nil
}

1
weed/shell/commands.go
View file

@ -14,6 +14,7 @@ type ShellOptions struct {
type commandEnv struct {
env map[string]string
masterClient *wdclient.MasterClient
option ShellOptions
}
type command interface {

1
weed/shell/shell_liner.go
View file

@ -36,6 +36,7 @@ func RunShell(options ShellOptions) {
env: make(map[string]string),
masterClient: wdclient.NewMasterClient(context.Background(),
options.GrpcDialOption, "shell", strings.Split(*options.Masters, ",")),
option: options,
}
go commandEnv.masterClient.KeepConnectedToMaster()

35
weed/storage/needle_read_write.go
View file

@ -182,10 +182,10 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
case Version1:
n.Data = bytes[NeedleEntrySize : NeedleEntrySize+size]
case Version2, Version3:
n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
err = n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
}
if size == 0 {
return nil
if size == 0 || err != nil {
return err
}
checksum := util.BytesToUint32(bytes[NeedleEntrySize+size : NeedleEntrySize+size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data)
@ -206,15 +206,13 @@ func (n *Needle) ParseNeedleHeader(bytes []byte) {
n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize : NeedleEntrySize])
}
func (n *Needle) readNeedleDataVersion2(bytes []byte) {
func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
index, lenBytes := 0, len(bytes)
if index < lenBytes {
n.DataSize = util.BytesToUint32(bytes[index : index+4])
index = index + 4
if int(n.DataSize)+index > lenBytes {
// this if clause is due to bug #87 and #93, fixed in v0.69
// remove this clause later
return
return fmt.Errorf("index out of range %d", 1)
}
n.Data = bytes[index : index+int(n.DataSize)]
index = index + int(n.DataSize)
@ -224,30 +222,49 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) {
if index < lenBytes && n.HasName() {
n.NameSize = uint8(bytes[index])
index = index + 1
if int(n.NameSize)+index > lenBytes {
return fmt.Errorf("index out of range %d", 2)
}
n.Name = bytes[index : index+int(n.NameSize)]
index = index + int(n.NameSize)
}
if index < lenBytes && n.HasMime() {
n.MimeSize = uint8(bytes[index])
index = index + 1
if int(n.MimeSize)+index > lenBytes {
return fmt.Errorf("index out of range %d", 3)
}
n.Mime = bytes[index : index+int(n.MimeSize)]
index = index + int(n.MimeSize)
}
if index < lenBytes && n.HasLastModifiedDate() {
if LastModifiedBytesLength+index > lenBytes {
return fmt.Errorf("index out of range %d", 4)
}
n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
index = index + LastModifiedBytesLength
}
if index < lenBytes && n.HasTtl() {
if TtlBytesLength+index > lenBytes {
return fmt.Errorf("index out of range %d", 5)
}
n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
index = index + TtlBytesLength
}
if index < lenBytes && n.HasPairs() {
if 2+index > lenBytes {
return fmt.Errorf("index out of range %d", 6)
}
n.PairsSize = util.BytesToUint16(bytes[index : index+2])
index += 2
if int(n.PairsSize)+index > lenBytes {
return fmt.Errorf("index out of range %d", 7)
}
end := index + int(n.PairsSize)
n.Pairs = bytes[index:end]
index = end
}
return nil
}
func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength int64, err error) {
@ -300,11 +317,11 @@ func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyL
if _, err = r.ReadAt(bytes, offset); err != nil {
return
}
n.readNeedleDataVersion2(bytes[0:n.Size])
err = n.readNeedleDataVersion2(bytes[0:n.Size])
n.Checksum = NewCRC(n.Data)
if version == Version3 {
tsOffset := n.Size+NeedleChecksumSize
tsOffset := n.Size + NeedleChecksumSize
n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize])
}
default:

4
weed/storage/store.go
View file

@ -121,7 +121,9 @@ func (s *Store) Status() []*VolumeInfo {
DeleteCount: v.nm.DeletedCount(),
DeletedByteCount: v.nm.DeletedSize(),
ReadOnly: v.readOnly,
Ttl: v.Ttl}
Ttl: v.Ttl,
CompactRevision: uint32(v.CompactRevision),
}
stats = append(stats, s)
}
location.RUnlock()

1
weed/storage/volume.go
View file

@ -148,5 +148,6 @@ func (v *Volume) ToVolumeInformationMessage() *master_pb.VolumeInformationMessag
ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
Version: uint32(v.Version()),
Ttl: v.Ttl.ToUint32(),
CompactRevision: uint32(v.SuperBlock.CompactRevision),
}
}

3
weed/storage/volume_info.go
View file

@ -18,6 +18,7 @@ type VolumeInfo struct {
DeleteCount int
DeletedByteCount uint64
ReadOnly bool
CompactRevision uint32
}
func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err error) {
@ -30,6 +31,7 @@ func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err er
DeletedByteCount: m.DeletedByteCount,
ReadOnly: m.ReadOnly,
Version: Version(m.Version),
CompactRevision: m.CompactRevision,
}
rp, e := NewReplicaPlacementFromByte(byte(m.ReplicaPlacement))
if e != nil {
@ -57,6 +59,7 @@ func (vi VolumeInfo) ToVolumeInformationMessage() *master_pb.VolumeInformationMe
ReplicaPlacement: uint32(vi.ReplicaPlacement.Byte()),
Version: uint32(vi.Version),
Ttl: vi.Ttl.ToUint32(),
CompactRevision: vi.CompactRevision,
}
}

5
weed/storage/volume_vacuum.go
View file

@ -335,7 +335,10 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
}
n := new(Needle)
n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version())
err := n.ReadData(v.dataFile, int64(offset)*NeedlePaddingSize, size, v.Version())
if err != nil {
return nil
}
if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) {
return nil

11
weed/util/grpc_client_server.go
View file

@ -81,7 +81,7 @@ func WithCachedGrpcClient(ctx context.Context, fn func(*grpc.ClientConn) error,
return err
}
func ParseServerToGrpcAddress(server string, optionalGrpcPort int) (serverGrpcAddress string, err error) {
func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
hostnameAndPort := strings.Split(server, ":")
if len(hostnameAndPort) != 2 {
return "", fmt.Errorf("server should have hostname:port format: %v", hostnameAndPort)
@ -93,22 +93,19 @@ func ParseServerToGrpcAddress(server string, optionalGrpcPort int) (serverGrpcAd
}
grpcPort := int(port) + 10000
if optionalGrpcPort != 0 {
grpcPort = optionalGrpcPort
}
return fmt.Sprintf("%s:%d", hostnameAndPort[0], grpcPort), nil
}
func ServerToGrpcAddress(server string, defaultGrpcPort int) (serverGrpcAddress string) {
func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
hostnameAndPort := strings.Split(server, ":")
if len(hostnameAndPort) != 2 {
return fmt.Sprintf("%s:%d", server, defaultGrpcPort)
return fmt.Sprintf("unexpected server address: %s", server)
}
port, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
if parseErr != nil {
return fmt.Sprintf("%s:%d", hostnameAndPort[0], defaultGrpcPort)
return fmt.Sprintf("failed to parse port for %s:%s", hostnameAndPort[0], hostnameAndPort[1])
}
grpcPort := int(port) + 10000

8
weed/wdclient/masterclient.go
View file

@ -101,7 +101,7 @@ func (mc *MasterClient) tryAllMasters() {
func withMasterClient(ctx context.Context, master string, grpcDialOption grpc.DialOption, fn func(ctx context.Context, client master_pb.SeaweedClient) error) error {
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master, 0)
masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master)
if parseErr != nil {
return fmt.Errorf("failed to parse master grpc %v", master)
}
@ -116,3 +116,9 @@ func withMasterClient(ctx context.Context, master string, grpcDialOption grpc.Di
return fn(ctx, client)
}
func (mc *MasterClient) WithClient(ctx context.Context, fn func(client master_pb.SeaweedClient) error) error {
return withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
return fn(client)
})
}

31
weed/wdclient/masterclient_collection.go
View file

@ -1,31 +0,0 @@
package wdclient
import (
"context"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)
func (mc *MasterClient) CollectionDelete(ctx context.Context, collection string) error {
return withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
Name: collection,
})
return err
})
}
func (mc *MasterClient) CollectionList(ctx context.Context) (resp *master_pb.CollectionListResponse, err error) {
err = withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
resp, err = client.CollectionList(ctx, &master_pb.CollectionListRequest{})
return err
})
return
}
func (mc *MasterClient) VolumeList(ctx context.Context) (resp *master_pb.VolumeListResponse, err error) {
err = withMasterClient(ctx, mc.currentMaster, mc.grpcDialOption, func(ctx context.Context, client master_pb.SeaweedClient) error {
resp, err = client.VolumeList(ctx, &master_pb.VolumeListRequest{})
return err
})
return
}

5
weed/wdclient/vid_map.go
View file

@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
)
@ -19,11 +20,13 @@ type Location struct {
type vidMap struct {
sync.RWMutex
vid2Locations map[uint32][]Location
r *rand.Rand
}
func newVidMap() vidMap {
return vidMap{
vid2Locations: make(map[uint32][]Location),
r: rand.New(rand.NewSource(time.Now().UnixNano())),
}
}
@ -39,7 +42,7 @@ func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrl string, err error
return "", fmt.Errorf("volume %d not found", id)
}
return locations[rand.Intn(len(locations))].Url, nil
return locations[vc.r.Intn(len(locations))].Url, nil
}
func (vc *vidMap) LookupFileId(fileId string) (fullUrl string, err error) {