gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

fs.configure: configurable volume growth

Browse source
This commit is contained in:
Chris Lu 2020-11-17 01:00:02 -08:00
parent b04375a3d1
commit dc304342b2
8 changed files with 80 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
other/java/client/src/main/proto/filer.proto
View file

@ -364,6 +364,7 @@ message FilerConf {
}
DiskType disk_type = 5;
bool fsync = 6;
uint32 volume_growth_count = 7;
}
repeated PathConf locations = 2;
}

3
weed/filer/filer_conf.go
View file

@ -122,6 +122,9 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
a.DiskType = b.DiskType
}
a.Fsync = b.Fsync || a.Fsync
if b.VolumeGrowthCount > 0 {
a.VolumeGrowthCount = b.VolumeGrowthCount
}
}
func (fc *FilerConf) ToProto() *filer_pb.FilerConf {

2
weed/filer/filer_notify_append.go
View file

@ -53,7 +53,7 @@ func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.Assi
Count: 1,
Collection: util.Nvl(f.metaLogCollection, rule.Collection),
Replication: util.Nvl(f.metaLogReplication, rule.Replication),
WritableVolumeCount: 1,
WritableVolumeCount: rule.VolumeGrowthCount,
}
assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)

3
weed/operation/assign_file_id.go
View file

@ -110,6 +110,7 @@ type StorageOption struct {
Rack string
TtlSeconds int32
Fsync bool
VolumeGrowthCount uint32
}
func (so *StorageOption) TtlString() string {
@ -124,6 +125,7 @@ func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, a
Ttl: so.TtlString(),
DataCenter: so.DataCenter,
Rack: so.Rack,
WritableVolumeCount: so.VolumeGrowthCount,
}
if so.DataCenter != "" || so.Rack != "" {
altRequest = &VolumeAssignRequest{
@ -133,6 +135,7 @@ func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, a
Ttl: so.TtlString(),
DataCenter: "",
Rack: "",
WritableVolumeCount: so.VolumeGrowthCount,
}
}
return

1
weed/pb/filer.proto
View file

@ -364,6 +364,7 @@ message FilerConf {
}
DiskType disk_type = 5;
bool fsync = 6;
uint32 volume_growth_count = 7;
}
repeated PathConf locations = 2;
}

17
weed/pb/filer_pb/filer.pb.go
View file

@ -3175,6 +3175,7 @@ type FilerConf_PathConf struct {
Ttl string `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"`
DiskType FilerConf_PathConf_DiskType `protobuf:"varint,5,opt,name=disk_type,json=diskType,proto3,enum=filer_pb.FilerConf_PathConf_DiskType" json:"disk_type,omitempty"`
Fsync bool `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"`
VolumeGrowthCount uint32 `protobuf:"varint,7,opt,name=volume_growth_count,json=volumeGrowthCount,proto3" json:"volume_growth_count,omitempty"`
}
func (x *FilerConf_PathConf) Reset() {
@ -3251,6 +3252,13 @@ func (x *FilerConf_PathConf) GetFsync() bool {
return false
}
func (x *FilerConf_PathConf) GetVolumeGrowthCount() uint32 {
if x != nil {
return x.VolumeGrowthCount
}
return 0
}
var File_filer_proto protoreflect.FileDescriptor
var file_filer_proto_rawDesc = []byte{
@ -3625,13 +3633,13 @@ var file_filer_proto_rawDesc = []byte{
0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xed, 0x02, 0x0a, 0x09, 0x46, 0x69,
0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x9d, 0x03, 0x0a, 0x09, 0x46, 0x69,
0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02,
0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f,
0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x89, 0x02,
0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xb9, 0x02,
0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f,
0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65,
@ -3646,7 +3654,10 @@ var file_filer_proto_rawDesc = []byte{
0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70,
0x65, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66,
0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, 0x6e,
0x63, 0x22, 0x26, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a,
0x63, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x77,
0x74, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11,
0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x43, 0x6f, 0x75, 0x6e,
0x74, 0x22, 0x26, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a,
0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x01,
0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x02, 0x32, 0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65,
0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f,

1
weed/server/filer_server_handlers_write.go
View file

@ -141,6 +141,7 @@ func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication
Rack: util.Nvl(rack, fs.option.Rack),
TtlSeconds: ttlSeconds,
Fsync: fsync || rule.Fsync,
VolumeGrowthCount: rule.VolumeGrowthCount,
}
}

21
weed/shell/command_fs_configure.go
View file

@ -11,6 +11,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/util"
)
@ -35,6 +36,9 @@ func (c *commandFsConfigure) Help() string {
fs.configure -locationPrfix=/my/folder -collection=abc
fs.configure -locationPrfix=/my/folder -collection=abc -ttl=7d
# example: configure adding only 1 physical volume for each bucket collection
fs.configure -locationPrfix=/buckets/ -volumeGrowthCount=1
# apply the changes
fs.configure -locationPrfix=/my/folder -collection=abc -apply
@ -52,6 +56,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io
replication := fsConfigureCommand.String("replication", "", "assign writes with this replication")
ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl")
fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes")
volumeGrowthCount := fsConfigureCommand.Int("volumeGrowthCount", 0, "the number of physical volumes to add if no writable volumes")
isDelete := fsConfigureCommand.Bool("delete", false, "delete the configuration by locationPrefix")
apply := fsConfigureCommand.Bool("apply", false, "update and apply filer configuration")
if err = fsConfigureCommand.Parse(args); err != nil {
@ -88,10 +93,26 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io
Replication: *replication,
Ttl: *ttl,
Fsync: *fsync,
VolumeGrowthCount: uint32(*volumeGrowthCount),
}
// check collection
if *collection != "" && strings.HasPrefix(*locationPrefix, "/buckets/") {
return fmt.Errorf("one s3 bucket goes to one collection and not customizable.")
}
// check replication
if *replication != "" {
rp, err := super_block.NewReplicaPlacementFromString(*replication)
if err != nil {
return fmt.Errorf("parse replication %s: %v", *replication, err)
}
if *volumeGrowthCount % rp.GetCopyCount() != 0 {
return fmt.Errorf("volumeGrowthCount %d should be devided by replication copy count %d", *volumeGrowthCount, rp.GetCopyCount())
}
}
// save it
if *isDelete {
fc.DeleteLocationConf(*locationPrefix)
} else {