From db94a41f9e28e620b7527d9cca51f9a052a81184 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 May 2019 23:23:19 -0700 Subject: [PATCH] mount/unmount ec shards --- weed/pb/volume_server.proto | 19 + weed/pb/volume_server_pb/volume_server.pb.go | 360 +++++++++++++------ weed/server/volume_grpc_erasure_coding.go | 39 ++ weed/shell/command_ec_encode.go | 35 +- weed/storage/disk_location_ec.go | 85 ++++- weed/storage/erasure_coding/ec_volume.go | 8 +- weed/storage/store.go | 1 + weed/storage/store_ec.go | 58 +++ 8 files changed, 471 insertions(+), 134 deletions(-) diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 5af0e8225..0f21c8ba5 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -54,6 +54,10 @@ service VolumeServer { } rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) { } + rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) { + } + rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) { + } } @@ -223,6 +227,21 @@ message VolumeEcShardsDeleteRequest { message VolumeEcShardsDeleteResponse { } +message VolumeEcShardsMountRequest { + uint32 volume_id = 1; + string collection = 2; + repeated uint32 ec_indexes = 3; +} +message VolumeEcShardsMountResponse { +} + +message VolumeEcShardsUnmountRequest { + uint32 volume_id = 1; + repeated uint32 ec_indexes = 3; +} +message VolumeEcShardsUnmountResponse { +} + message ReadVolumeFileStatusRequest { uint32 volume_id = 1; } diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 4b0c10e0b..f572fe401 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -49,6 +49,10 @@ It has these top-level messages: VolumeEcShardsCopyResponse VolumeEcShardsDeleteRequest VolumeEcShardsDeleteResponse + VolumeEcShardsMountRequest + VolumeEcShardsMountResponse + VolumeEcShardsUnmountRequest + VolumeEcShardsUnmountResponse ReadVolumeFileStatusRequest ReadVolumeFileStatusResponse DiskStatus @@ -892,6 +896,78 @@ func (m *VolumeEcShardsDeleteResponse) String() string { return proto func (*VolumeEcShardsDeleteResponse) ProtoMessage() {} func (*VolumeEcShardsDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{39} } +type VolumeEcShardsMountRequest struct { + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` + EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"` +} + +func (m *VolumeEcShardsMountRequest) Reset() { *m = VolumeEcShardsMountRequest{} } +func (m *VolumeEcShardsMountRequest) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsMountRequest) ProtoMessage() {} +func (*VolumeEcShardsMountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{40} } + +func (m *VolumeEcShardsMountRequest) GetVolumeId() uint32 { + if m != nil { + return m.VolumeId + } + return 0 +} + +func (m *VolumeEcShardsMountRequest) GetCollection() string { + if m != nil { + return m.Collection + } + return "" +} + +func (m *VolumeEcShardsMountRequest) GetEcIndexes() []uint32 { + if m != nil { + return m.EcIndexes + } + return nil +} + +type VolumeEcShardsMountResponse struct { +} + +func (m *VolumeEcShardsMountResponse) Reset() { *m = VolumeEcShardsMountResponse{} } +func (m *VolumeEcShardsMountResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsMountResponse) ProtoMessage() {} +func (*VolumeEcShardsMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} } + +type VolumeEcShardsUnmountRequest struct { + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"` +} + +func (m *VolumeEcShardsUnmountRequest) Reset() { *m = VolumeEcShardsUnmountRequest{} } +func (m *VolumeEcShardsUnmountRequest) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsUnmountRequest) ProtoMessage() {} +func (*VolumeEcShardsUnmountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{42} } + +func (m *VolumeEcShardsUnmountRequest) GetVolumeId() uint32 { + if m != nil { + return m.VolumeId + } + return 0 +} + +func (m *VolumeEcShardsUnmountRequest) GetEcIndexes() []uint32 { + if m != nil { + return m.EcIndexes + } + return nil +} + +type VolumeEcShardsUnmountResponse struct { +} + +func (m *VolumeEcShardsUnmountResponse) Reset() { *m = VolumeEcShardsUnmountResponse{} } +func (m *VolumeEcShardsUnmountResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsUnmountResponse) ProtoMessage() {} +func (*VolumeEcShardsUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{43} } + type ReadVolumeFileStatusRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` } @@ -899,7 +975,7 @@ type ReadVolumeFileStatusRequest struct { func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} } func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) } func (*ReadVolumeFileStatusRequest) ProtoMessage() {} -func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{40} } +func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{44} } func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 { if m != nil { @@ -922,7 +998,7 @@ type ReadVolumeFileStatusResponse struct { func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} } func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) } func (*ReadVolumeFileStatusResponse) ProtoMessage() {} -func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} } +func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{45} } func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 { if m != nil { @@ -990,7 +1066,7 @@ type DiskStatus struct { func (m *DiskStatus) Reset() { *m = DiskStatus{} } func (m *DiskStatus) String() string { return proto.CompactTextString(m) } func (*DiskStatus) ProtoMessage() {} -func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{42} } +func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{46} } func (m *DiskStatus) GetDir() string { if m != nil { @@ -1033,7 +1109,7 @@ type MemStatus struct { func (m *MemStatus) Reset() { *m = MemStatus{} } func (m *MemStatus) String() string { return proto.CompactTextString(m) } func (*MemStatus) ProtoMessage() {} -func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{43} } +func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{47} } func (m *MemStatus) GetGoroutines() int32 { if m != nil { @@ -1125,6 +1201,10 @@ func init() { proto.RegisterType((*VolumeEcShardsCopyResponse)(nil), "volume_server_pb.VolumeEcShardsCopyResponse") proto.RegisterType((*VolumeEcShardsDeleteRequest)(nil), "volume_server_pb.VolumeEcShardsDeleteRequest") proto.RegisterType((*VolumeEcShardsDeleteResponse)(nil), "volume_server_pb.VolumeEcShardsDeleteResponse") + proto.RegisterType((*VolumeEcShardsMountRequest)(nil), "volume_server_pb.VolumeEcShardsMountRequest") + proto.RegisterType((*VolumeEcShardsMountResponse)(nil), "volume_server_pb.VolumeEcShardsMountResponse") + proto.RegisterType((*VolumeEcShardsUnmountRequest)(nil), "volume_server_pb.VolumeEcShardsUnmountRequest") + proto.RegisterType((*VolumeEcShardsUnmountResponse)(nil), "volume_server_pb.VolumeEcShardsUnmountResponse") proto.RegisterType((*ReadVolumeFileStatusRequest)(nil), "volume_server_pb.ReadVolumeFileStatusRequest") proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse") proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus") @@ -1165,6 +1245,8 @@ type VolumeServerClient interface { VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error) VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error) VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) + VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error) + VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) } type volumeServerClient struct { @@ -1424,6 +1506,24 @@ func (c *volumeServerClient) VolumeEcShardsDelete(ctx context.Context, in *Volum return out, nil } +func (c *volumeServerClient) VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error) { + out := new(VolumeEcShardsMountResponse) + err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsMount", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) { + out := new(VolumeEcShardsUnmountResponse) + err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for VolumeServer service type VolumeServerServer interface { @@ -1450,6 +1550,8 @@ type VolumeServerServer interface { VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error) VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error) VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) + VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) + VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) } func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) { @@ -1825,6 +1927,42 @@ func _VolumeServer_VolumeEcShardsDelete_Handler(srv interface{}, ctx context.Con return interceptor(ctx, in, info, handler) } +func _VolumeServer_VolumeEcShardsMount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsMountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsMount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsMount", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsMount(ctx, req.(*VolumeEcShardsMountRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeEcShardsUnmount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeEcShardsUnmountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeEcShardsUnmount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeEcShardsUnmount(ctx, req.(*VolumeEcShardsUnmountRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _VolumeServer_serviceDesc = grpc.ServiceDesc{ ServiceName: "volume_server_pb.VolumeServer", HandlerType: (*VolumeServerServer)(nil), @@ -1897,6 +2035,14 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ MethodName: "VolumeEcShardsDelete", Handler: _VolumeServer_VolumeEcShardsDelete_Handler, }, + { + MethodName: "VolumeEcShardsMount", + Handler: _VolumeServer_VolumeEcShardsMount_Handler, + }, + { + MethodName: "VolumeEcShardsUnmount", + Handler: _VolumeServer_VolumeEcShardsUnmount_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -1921,105 +2067,109 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1598 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x58, 0x5f, 0x6f, 0xd4, 0x56, - 0x16, 0x8f, 0x99, 0x49, 0x32, 0x39, 0x93, 0x40, 0x72, 0x13, 0x92, 0x89, 0x43, 0x42, 0x30, 0x2c, - 0x0c, 0x7f, 0x36, 0xb0, 0xa0, 0xdd, 0x65, 0x77, 0x1f, 0xb6, 0x10, 0x68, 0x1b, 0xa9, 0x80, 0xe4, - 0x00, 0x6a, 0x55, 0x24, 0xeb, 0xc6, 0x3e, 0x21, 0x56, 0x3c, 0xf6, 0xe0, 0x7b, 0x9d, 0x26, 0x55, - 0xfb, 0xd4, 0xf6, 0x0b, 0x54, 0xea, 0x07, 0xa8, 0xfa, 0xd6, 0x87, 0xbe, 0xf6, 0x53, 0xf5, 0x13, - 0xf4, 0xa5, 0xba, 0x7f, 0xec, 0x19, 0x8f, 0xed, 0x8c, 0x29, 0x48, 0x7d, 0xbb, 0x73, 0xee, 0xf9, - 0xef, 0x73, 0xce, 0x3d, 0xbf, 0x81, 0xc5, 0xa3, 0x28, 0x48, 0x7a, 0xe8, 0x30, 0x8c, 0x8f, 0x30, - 0xde, 0xea, 0xc7, 0x11, 0x8f, 0xc8, 0x7c, 0x8e, 0xe8, 0xf4, 0xf7, 0xac, 0xdb, 0x40, 0x1e, 0x52, - 0xee, 0x1e, 0x3c, 0xc2, 0x00, 0x39, 0xda, 0xf8, 0x26, 0x41, 0xc6, 0xc9, 0x2a, 0xb4, 0xf6, 0xfd, - 0x00, 0x1d, 0xdf, 0x63, 0x1d, 0x63, 0xb3, 0xd1, 0x9d, 0xb1, 0xa7, 0xc5, 0xef, 0x1d, 0x8f, 0x59, - 0xcf, 0x60, 0x31, 0x27, 0xc0, 0xfa, 0x51, 0xc8, 0x90, 0xdc, 0x87, 0xe9, 0x18, 0x59, 0x12, 0x70, - 0x25, 0xd0, 0xbe, 0xbb, 0xb1, 0x35, 0x6a, 0x6b, 0x2b, 0x13, 0x49, 0x02, 0x6e, 0xa7, 0xec, 0x96, - 0x0f, 0xb3, 0xc3, 0x17, 0x64, 0x05, 0xa6, 0xb5, 0xed, 0x8e, 0xb1, 0x69, 0x74, 0x67, 0xec, 0x29, - 0x65, 0x9a, 0x2c, 0xc3, 0x14, 0xe3, 0x94, 0x27, 0xac, 0x73, 0x66, 0xd3, 0xe8, 0x4e, 0xda, 0xfa, - 0x17, 0x59, 0x82, 0x49, 0x8c, 0xe3, 0x28, 0xee, 0x34, 0x24, 0xbb, 0xfa, 0x41, 0x08, 0x34, 0x99, - 0xff, 0x25, 0x76, 0x9a, 0x9b, 0x46, 0x77, 0xce, 0x96, 0x67, 0x6b, 0x1a, 0x26, 0x1f, 0xf7, 0xfa, - 0xfc, 0xc4, 0xfa, 0x37, 0x74, 0x5e, 0x52, 0x37, 0x49, 0x7a, 0x2f, 0xa5, 0x8f, 0xdb, 0x07, 0xe8, - 0x1e, 0xa6, 0xb1, 0xaf, 0xc1, 0x8c, 0xf6, 0x5c, 0x7b, 0x30, 0x67, 0xb7, 0x14, 0x61, 0xc7, 0xb3, - 0x3e, 0x80, 0xd5, 0x12, 0x41, 0x9d, 0x83, 0xcb, 0x30, 0xf7, 0x9a, 0xc6, 0x7b, 0xf4, 0x35, 0x3a, - 0x31, 0xe5, 0x7e, 0x24, 0xa5, 0x0d, 0x7b, 0x56, 0x13, 0x6d, 0x41, 0xb3, 0x3e, 0x07, 0x33, 0xa7, - 0x21, 0xea, 0xf5, 0xa9, 0xcb, 0xeb, 0x18, 0x27, 0x9b, 0xd0, 0xee, 0xc7, 0x48, 0x83, 0x20, 0x72, - 0x29, 0x47, 0x99, 0x85, 0x86, 0x3d, 0x4c, 0xb2, 0xd6, 0x61, 0xad, 0x54, 0xb9, 0x72, 0xd0, 0xba, - 0x3f, 0xe2, 0x7d, 0xd4, 0xeb, 0xf9, 0xb5, 0x4c, 0x5b, 0x17, 0x0a, 0x5e, 0x4b, 0x49, 0xad, 0xf7, - 0x3f, 0x23, 0xb7, 0x01, 0xd2, 0x30, 0xe9, 0xd7, 0x52, 0x3c, 0xea, 0x71, 0x2a, 0x9a, 0x69, 0x5e, - 0x51, 0xc5, 0xb1, 0x1d, 0x05, 0x01, 0xba, 0xdc, 0x8f, 0xc2, 0x54, 0xed, 0x06, 0x80, 0x9b, 0x11, - 0x75, 0xa9, 0x0c, 0x51, 0x2c, 0x13, 0x3a, 0x45, 0x51, 0xad, 0xf6, 0x67, 0x03, 0xce, 0x3f, 0xd0, - 0x49, 0x53, 0x86, 0x6b, 0x7d, 0x80, 0xbc, 0xc9, 0x33, 0xa3, 0x26, 0x47, 0x3f, 0x50, 0xa3, 0xf0, - 0x81, 0x04, 0x47, 0x8c, 0xfd, 0xc0, 0x77, 0xa9, 0x54, 0xd1, 0x94, 0x2a, 0x86, 0x49, 0x64, 0x1e, - 0x1a, 0x9c, 0x07, 0x9d, 0x49, 0x79, 0x23, 0x8e, 0x56, 0x07, 0x96, 0x47, 0x7d, 0xd5, 0x61, 0xfc, - 0x0b, 0x56, 0x14, 0x65, 0xf7, 0x24, 0x74, 0x77, 0x65, 0x37, 0xd4, 0x4a, 0xfa, 0xef, 0x06, 0x74, - 0x8a, 0x82, 0xba, 0x8a, 0xdf, 0x35, 0x03, 0x6f, 0x1b, 0x1f, 0xb9, 0x08, 0x6d, 0x4e, 0xfd, 0xc0, - 0x89, 0xf6, 0xf7, 0x19, 0xf2, 0xce, 0xd4, 0xa6, 0xd1, 0x6d, 0xda, 0x20, 0x48, 0xcf, 0x24, 0x85, - 0x5c, 0x87, 0x79, 0x57, 0x55, 0xb2, 0x13, 0xe3, 0x91, 0xcf, 0x84, 0xe6, 0x69, 0xe9, 0xd8, 0x39, - 0x37, 0xad, 0x70, 0x45, 0x26, 0x16, 0xcc, 0xf9, 0xde, 0xb1, 0x23, 0x07, 0x88, 0x6c, 0xff, 0x96, - 0xd4, 0xd6, 0xf6, 0xbd, 0xe3, 0x0f, 0xfd, 0x00, 0x77, 0xc5, 0x14, 0x78, 0x09, 0x17, 0x54, 0xf0, - 0x3b, 0xa1, 0x1b, 0x63, 0x0f, 0x43, 0x4e, 0x83, 0xed, 0xa8, 0x7f, 0x52, 0xab, 0x04, 0x56, 0xa1, - 0xc5, 0xfc, 0xd0, 0x45, 0x27, 0x54, 0x63, 0xa8, 0x69, 0x4f, 0xcb, 0xdf, 0x4f, 0x99, 0xf5, 0x10, - 0xd6, 0x2b, 0xf4, 0xea, 0xcc, 0x5e, 0x82, 0x59, 0xe9, 0x98, 0x1b, 0x85, 0x1c, 0x43, 0x2e, 0x75, - 0xcf, 0xda, 0x6d, 0x41, 0xdb, 0x56, 0x24, 0xeb, 0x1f, 0x40, 0x94, 0x8e, 0x27, 0x51, 0x12, 0xd6, - 0x6b, 0xcd, 0xf3, 0xb0, 0x98, 0x13, 0xd1, 0xb5, 0x71, 0x0f, 0x96, 0x14, 0xf9, 0x45, 0xd8, 0xab, - 0xad, 0x6b, 0x05, 0xce, 0x8f, 0x08, 0x69, 0x6d, 0x77, 0x53, 0x23, 0xf9, 0x77, 0xe2, 0x54, 0x65, - 0xcb, 0xa9, 0x07, 0xf9, 0xa7, 0xc2, 0xfa, 0xc5, 0x80, 0x85, 0x74, 0x8c, 0xd4, 0xcc, 0xfa, 0x5b, - 0x96, 0x5d, 0xa3, 0xb2, 0xec, 0x9a, 0x83, 0xb2, 0xeb, 0xc2, 0x3c, 0x8b, 0x92, 0xd8, 0x45, 0xc7, - 0xa3, 0x9c, 0x3a, 0x61, 0xe4, 0xa1, 0xae, 0xca, 0xb3, 0x8a, 0xfe, 0x88, 0x72, 0xfa, 0x34, 0xf2, - 0xd0, 0xfa, 0x7f, 0xfa, 0x51, 0x72, 0x5f, 0xf3, 0x3a, 0x2c, 0x04, 0x94, 0x71, 0x87, 0xf6, 0xfb, - 0x18, 0x7a, 0x0e, 0xe5, 0xa2, 0x24, 0x0c, 0x59, 0x12, 0x67, 0xc5, 0xc5, 0x03, 0x49, 0x7f, 0xc0, - 0x9f, 0x32, 0xeb, 0x7b, 0x03, 0xce, 0x09, 0x59, 0x51, 0x82, 0xb5, 0xe2, 0x9d, 0x87, 0x06, 0x1e, - 0x73, 0x1d, 0xa8, 0x38, 0x92, 0xdb, 0xb0, 0xa8, 0x6b, 0xdd, 0x8f, 0xc2, 0x41, 0x1b, 0x34, 0xa4, - 0x20, 0x19, 0x5c, 0x65, 0x9d, 0x70, 0x11, 0xda, 0x8c, 0x47, 0xfd, 0xb4, 0xab, 0x9a, 0xaa, 0xab, - 0x04, 0x49, 0x75, 0x95, 0xf5, 0x4f, 0x98, 0x1f, 0xf8, 0x54, 0xbf, 0x42, 0xbf, 0x31, 0xd2, 0xa1, - 0xf3, 0x9c, 0xfa, 0xc1, 0x2e, 0x86, 0x1e, 0xc6, 0xef, 0xd8, 0x39, 0xe4, 0x0e, 0x2c, 0xf9, 0x5e, - 0x80, 0x0e, 0xf7, 0x7b, 0x18, 0x25, 0xdc, 0x61, 0xe8, 0x46, 0xa1, 0xc7, 0xd2, 0xe8, 0xc4, 0xdd, - 0x73, 0x75, 0xb5, 0xab, 0x6e, 0xac, 0x6f, 0xb3, 0x09, 0x36, 0xec, 0xc5, 0xe0, 0x1d, 0x0e, 0x11, - 0x85, 0xc2, 0x03, 0xa4, 0x1e, 0xc6, 0x3a, 0x8c, 0x59, 0x45, 0xfc, 0x58, 0xd2, 0x44, 0x7e, 0x34, - 0xd3, 0x5e, 0xe4, 0x9d, 0x48, 0x8f, 0x66, 0x6d, 0x50, 0xa4, 0x87, 0x91, 0x77, 0x22, 0x47, 0x09, - 0x73, 0xe4, 0x27, 0x76, 0x0f, 0x92, 0xf0, 0x50, 0x7a, 0xd3, 0xb2, 0xdb, 0x3e, 0xfb, 0x84, 0x32, - 0xbe, 0x2d, 0x48, 0xd6, 0xaf, 0x06, 0xac, 0x0e, 0xdc, 0xb0, 0xd1, 0x45, 0xff, 0xe8, 0x2f, 0x48, - 0x87, 0x90, 0xd0, 0xb5, 0x9c, 0x5b, 0xba, 0x74, 0xb9, 0x13, 0x75, 0xa7, 0x27, 0xbe, 0xbc, 0x91, - 0x0f, 0x7a, 0x89, 0xe3, 0xba, 0x45, 0x5f, 0xa5, 0xa3, 0xec, 0xb1, 0xbb, 0x7b, 0x40, 0x63, 0x8f, - 0x7d, 0x84, 0x21, 0xc6, 0x94, 0xbf, 0x97, 0x67, 0xd2, 0xda, 0x84, 0x8d, 0x2a, 0xed, 0xda, 0xfe, - 0x8f, 0x59, 0x5e, 0x53, 0x96, 0xf7, 0x36, 0x2a, 0xd6, 0x01, 0xd0, 0x75, 0xfc, 0xd0, 0xc3, 0x63, - 0x14, 0x29, 0x6d, 0x74, 0xe7, 0xec, 0x19, 0x74, 0x77, 0x14, 0xe1, 0x2d, 0xa6, 0x42, 0x96, 0xc1, - 0xbc, 0x8b, 0x3a, 0x82, 0xef, 0x0c, 0x58, 0xcb, 0x5f, 0xd7, 0x9f, 0x9c, 0xe4, 0x06, 0x2c, 0xb0, - 0x83, 0x28, 0x09, 0x3c, 0xc7, 0x93, 0x42, 0x0e, 0xba, 0xc7, 0x32, 0x94, 0x96, 0x7d, 0x4e, 0x5d, - 0x28, 0x65, 0x8f, 0xdd, 0xe3, 0x31, 0xf1, 0x58, 0x1b, 0xe9, 0x63, 0x37, 0xea, 0x86, 0xf6, 0xf3, - 0xbf, 0xb0, 0x66, 0x23, 0xf5, 0x14, 0x8f, 0x7c, 0x22, 0xeb, 0xaf, 0x11, 0xbf, 0x9d, 0x81, 0x0b, - 0xe5, 0xc2, 0x75, 0x56, 0x89, 0xff, 0x81, 0x99, 0x3d, 0xd5, 0xa2, 0xd2, 0x19, 0xa7, 0xbd, 0x7e, - 0x56, 0xeb, 0xaa, 0x25, 0x56, 0xf4, 0xbb, 0xfd, 0x3c, 0xbd, 0x4f, 0x0b, 0xbe, 0xf0, 0xce, 0x37, - 0x0a, 0xef, 0xbc, 0x30, 0xe0, 0x51, 0x5e, 0x65, 0x40, 0x0d, 0xc4, 0x15, 0x8f, 0xf2, 0x2a, 0x03, - 0x99, 0xb0, 0x34, 0x30, 0xa9, 0x0c, 0x68, 0x7e, 0x69, 0x60, 0x1d, 0x40, 0x4f, 0xcb, 0x24, 0x4c, - 0xf7, 0x96, 0x19, 0x35, 0x2b, 0x93, 0xb0, 0x72, 0x64, 0x4f, 0x57, 0x8e, 0xec, 0x7c, 0xe9, 0xb6, - 0x0a, 0x7d, 0xf3, 0x29, 0xc0, 0x23, 0x9f, 0x1d, 0xaa, 0x24, 0x8b, 0x37, 0xc2, 0xf3, 0x63, 0xbd, - 0xf8, 0x8a, 0xa3, 0xa0, 0xd0, 0x20, 0xd0, 0xa9, 0x13, 0x47, 0x01, 0x82, 0x12, 0x86, 0x9e, 0xce, - 0x8e, 0x3c, 0x0b, 0xda, 0x7e, 0x8c, 0xa8, 0x13, 0x20, 0xcf, 0xd6, 0x4f, 0x06, 0xcc, 0x3c, 0xc1, - 0x9e, 0xd6, 0xbc, 0x01, 0xf0, 0x3a, 0x8a, 0xa3, 0x84, 0xfb, 0x21, 0xaa, 0x27, 0x6d, 0xd2, 0x1e, - 0xa2, 0xfc, 0x79, 0x3b, 0x12, 0x94, 0x61, 0xb0, 0xaf, 0x93, 0x29, 0xcf, 0x82, 0x76, 0x80, 0xb4, - 0xaf, 0xf3, 0x27, 0xcf, 0x02, 0xd2, 0x31, 0x4e, 0xdd, 0x43, 0x99, 0xac, 0xa6, 0xad, 0x7e, 0xdc, - 0xfd, 0x61, 0x01, 0x66, 0x87, 0x87, 0x18, 0x79, 0x05, 0xed, 0x21, 0x2c, 0x4a, 0xae, 0x14, 0x21, - 0x67, 0x11, 0xdb, 0x9a, 0x7f, 0x1b, 0xc3, 0xa5, 0x1b, 0x63, 0x82, 0x84, 0xb0, 0x50, 0xc0, 0x7a, - 0xe4, 0x46, 0x51, 0xba, 0x0a, 0x49, 0x9a, 0x37, 0x6b, 0xf1, 0x66, 0xf6, 0x38, 0x2c, 0x96, 0x80, - 0x37, 0x72, 0x6b, 0x8c, 0x96, 0x1c, 0x80, 0x34, 0xff, 0x5e, 0x93, 0x3b, 0xb3, 0xfa, 0x06, 0x48, - 0x11, 0xd9, 0x91, 0x9b, 0x63, 0xd5, 0x0c, 0x90, 0xa3, 0x79, 0xab, 0x1e, 0x73, 0x65, 0xa0, 0x0a, - 0xf3, 0x8d, 0x0d, 0x34, 0x87, 0x2a, 0xc7, 0x06, 0x3a, 0x02, 0x24, 0x27, 0xc8, 0x21, 0xcc, 0x8f, - 0xe2, 0x41, 0x72, 0xbd, 0xea, 0x4f, 0x8a, 0x02, 0xdc, 0x34, 0x6f, 0xd4, 0x61, 0xcd, 0x8c, 0x21, - 0x9c, 0xcd, 0x63, 0x36, 0x72, 0xad, 0x28, 0x5f, 0x8a, 0x40, 0xcd, 0xee, 0x78, 0xc6, 0xe1, 0x98, - 0x46, 0x71, 0x5c, 0x59, 0x4c, 0x15, 0x20, 0xb1, 0x2c, 0xa6, 0x2a, 0x58, 0x68, 0x4d, 0x90, 0xaf, - 0x52, 0x70, 0x30, 0x82, 0x6f, 0xc8, 0x56, 0x95, 0x9a, 0x72, 0x80, 0x65, 0xde, 0xae, 0xcd, 0x9f, - 0xda, 0xbe, 0x63, 0x88, 0x5e, 0x1f, 0x82, 0x39, 0x65, 0xbd, 0x5e, 0x04, 0x4e, 0x65, 0xbd, 0x5e, - 0x86, 0x95, 0x26, 0xc8, 0x1e, 0xcc, 0xe5, 0x80, 0x0f, 0xb9, 0x5a, 0x25, 0x99, 0x87, 0x53, 0xe6, - 0xb5, 0xb1, 0x7c, 0x99, 0x0d, 0x27, 0x9d, 0x5e, 0x7a, 0x5c, 0x55, 0x3a, 0x97, 0x9f, 0x57, 0x57, - 0xc7, 0xb1, 0x65, 0x06, 0x3e, 0x03, 0x18, 0xe0, 0x14, 0x72, 0xb9, 0x4a, 0x6e, 0xf8, 0x53, 0x5c, - 0x39, 0x9d, 0x29, 0x53, 0xfd, 0x05, 0x2c, 0x95, 0xbd, 0xf4, 0xa4, 0xa4, 0x0b, 0x4f, 0x59, 0x27, - 0xcc, 0xad, 0xba, 0xec, 0x99, 0xe1, 0x17, 0xd0, 0x4a, 0x51, 0x0a, 0xb9, 0x54, 0x94, 0x1e, 0x41, - 0x55, 0xa6, 0x75, 0x1a, 0xcb, 0x50, 0x35, 0xf5, 0xd2, 0xc6, 0x19, 0xc0, 0x87, 0xea, 0xc6, 0x29, - 0x00, 0x9d, 0xea, 0xc6, 0x29, 0xa2, 0x11, 0x69, 0xee, 0x4d, 0x8a, 0x20, 0x87, 0xb7, 0xed, 0xd2, - 0x21, 0x5b, 0x05, 0x26, 0x4a, 0x87, 0x6c, 0xf5, 0x02, 0x3f, 0x41, 0xbe, 0x86, 0xe5, 0xf2, 0x25, - 0x9b, 0x54, 0xb6, 0x5f, 0xc5, 0xb2, 0x6f, 0xde, 0xa9, 0x2f, 0x90, 0x7b, 0x56, 0x0a, 0xdb, 0x71, - 0x75, 0xc4, 0x25, 0x6b, 0x7e, 0x75, 0xc4, 0xa5, 0x0b, 0xb7, 0xac, 0xd1, 0xb2, 0x55, 0xb7, 0xac, - 0x46, 0x4f, 0xd9, 0xcc, 0xcd, 0xad, 0xba, 0xec, 0xa9, 0xe1, 0xbd, 0x29, 0xf9, 0xe7, 0xfa, 0xbd, - 0x3f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x1a, 0xed, 0xaf, 0x31, 0x73, 0x17, 0x00, 0x00, + // 1655 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x59, 0x5b, 0x4f, 0xdc, 0xd6, + 0x16, 0xc6, 0x99, 0x01, 0x86, 0x35, 0x90, 0xc0, 0x86, 0xc0, 0x60, 0x02, 0x21, 0x4e, 0x4e, 0x32, + 0xb9, 0x41, 0x4e, 0xa2, 0x73, 0x4e, 0xce, 0x39, 0x0f, 0x6d, 0x42, 0xd2, 0x16, 0xa9, 0x49, 0x24, + 0x93, 0x44, 0xbd, 0x44, 0xb2, 0x36, 0xf6, 0x22, 0x58, 0x78, 0xec, 0x89, 0xbd, 0x4d, 0x21, 0x6a, + 0x9f, 0xda, 0xaa, 0xef, 0xfd, 0x07, 0x55, 0xdf, 0xfa, 0xd0, 0xd7, 0xfe, 0xaa, 0xfe, 0x82, 0xbe, + 0x54, 0xfb, 0x62, 0xcf, 0xf8, 0xc6, 0x98, 0x26, 0x52, 0xdf, 0xf6, 0xac, 0xbd, 0xee, 0x5e, 0x97, + 0xfd, 0x01, 0xcc, 0x1f, 0x06, 0x5e, 0xdc, 0x43, 0x2b, 0xc2, 0xf0, 0x10, 0xc3, 0x8d, 0x7e, 0x18, + 0xb0, 0x80, 0xcc, 0x66, 0x88, 0x56, 0x7f, 0xd7, 0xd8, 0x04, 0xf2, 0x90, 0x32, 0x7b, 0xff, 0x11, + 0x7a, 0xc8, 0xd0, 0xc4, 0x37, 0x31, 0x46, 0x8c, 0x2c, 0x43, 0x6b, 0xcf, 0xf5, 0xd0, 0x72, 0x9d, + 0xa8, 0xa3, 0xad, 0x37, 0xba, 0x53, 0xe6, 0x24, 0xff, 0xbd, 0xed, 0x44, 0xc6, 0x33, 0x98, 0xcf, + 0x08, 0x44, 0xfd, 0xc0, 0x8f, 0x90, 0xdc, 0x87, 0xc9, 0x10, 0xa3, 0xd8, 0x63, 0x52, 0xa0, 0x7d, + 0x77, 0x6d, 0x23, 0x6f, 0x6b, 0x23, 0x15, 0x89, 0x3d, 0x66, 0x26, 0xec, 0x86, 0x0b, 0xd3, 0xc3, + 0x17, 0x64, 0x09, 0x26, 0x95, 0xed, 0x8e, 0xb6, 0xae, 0x75, 0xa7, 0xcc, 0x09, 0x69, 0x9a, 0x2c, + 0xc2, 0x44, 0xc4, 0x28, 0x8b, 0xa3, 0xce, 0x99, 0x75, 0xad, 0x3b, 0x6e, 0xaa, 0x5f, 0x64, 0x01, + 0xc6, 0x31, 0x0c, 0x83, 0xb0, 0xd3, 0x10, 0xec, 0xf2, 0x07, 0x21, 0xd0, 0x8c, 0xdc, 0xb7, 0xd8, + 0x69, 0xae, 0x6b, 0xdd, 0x19, 0x53, 0x9c, 0x8d, 0x49, 0x18, 0x7f, 0xdc, 0xeb, 0xb3, 0x63, 0xe3, + 0x3f, 0xd0, 0x79, 0x49, 0xed, 0x38, 0xee, 0xbd, 0x14, 0x3e, 0x6e, 0xed, 0xa3, 0x7d, 0x90, 0xc4, + 0xbe, 0x02, 0x53, 0xca, 0x73, 0xe5, 0xc1, 0x8c, 0xd9, 0x92, 0x84, 0x6d, 0xc7, 0xf8, 0x10, 0x96, + 0x4b, 0x04, 0x55, 0x0e, 0x2e, 0xc3, 0xcc, 0x6b, 0x1a, 0xee, 0xd2, 0xd7, 0x68, 0x85, 0x94, 0xb9, + 0x81, 0x90, 0xd6, 0xcc, 0x69, 0x45, 0x34, 0x39, 0xcd, 0xf8, 0x12, 0xf4, 0x8c, 0x86, 0xa0, 0xd7, + 0xa7, 0x36, 0xab, 0x63, 0x9c, 0xac, 0x43, 0xbb, 0x1f, 0x22, 0xf5, 0xbc, 0xc0, 0xa6, 0x0c, 0x45, + 0x16, 0x1a, 0xe6, 0x30, 0xc9, 0x58, 0x85, 0x95, 0x52, 0xe5, 0xd2, 0x41, 0xe3, 0x7e, 0xce, 0xfb, + 0xa0, 0xd7, 0x73, 0x6b, 0x99, 0x36, 0x2e, 0x14, 0xbc, 0x16, 0x92, 0x4a, 0xef, 0x7f, 0x73, 0xb7, + 0x1e, 0x52, 0x3f, 0xee, 0xd7, 0x52, 0x9c, 0xf7, 0x38, 0x11, 0x4d, 0x35, 0x2f, 0xc9, 0xe2, 0xd8, + 0x0a, 0x3c, 0x0f, 0x6d, 0xe6, 0x06, 0x7e, 0xa2, 0x76, 0x0d, 0xc0, 0x4e, 0x89, 0xaa, 0x54, 0x86, + 0x28, 0x86, 0x0e, 0x9d, 0xa2, 0xa8, 0x52, 0xfb, 0x8b, 0x06, 0xe7, 0x1f, 0xa8, 0xa4, 0x49, 0xc3, + 0xb5, 0x3e, 0x40, 0xd6, 0xe4, 0x99, 0xbc, 0xc9, 0xfc, 0x07, 0x6a, 0x14, 0x3e, 0x10, 0xe7, 0x08, + 0xb1, 0xef, 0xb9, 0x36, 0x15, 0x2a, 0x9a, 0x42, 0xc5, 0x30, 0x89, 0xcc, 0x42, 0x83, 0x31, 0xaf, + 0x33, 0x2e, 0x6e, 0xf8, 0xd1, 0xe8, 0xc0, 0x62, 0xde, 0x57, 0x15, 0xc6, 0xbf, 0x61, 0x49, 0x52, + 0x76, 0x8e, 0x7d, 0x7b, 0x47, 0x74, 0x43, 0xad, 0xa4, 0xff, 0xa1, 0x41, 0xa7, 0x28, 0xa8, 0xaa, + 0xf8, 0x5d, 0x33, 0x70, 0xda, 0xf8, 0xc8, 0x45, 0x68, 0x33, 0xea, 0x7a, 0x56, 0xb0, 0xb7, 0x17, + 0x21, 0xeb, 0x4c, 0xac, 0x6b, 0xdd, 0xa6, 0x09, 0x9c, 0xf4, 0x4c, 0x50, 0xc8, 0x75, 0x98, 0xb5, + 0x65, 0x25, 0x5b, 0x21, 0x1e, 0xba, 0x11, 0xd7, 0x3c, 0x29, 0x1c, 0x3b, 0x67, 0x27, 0x15, 0x2e, + 0xc9, 0xc4, 0x80, 0x19, 0xd7, 0x39, 0xb2, 0xc4, 0x00, 0x11, 0xed, 0xdf, 0x12, 0xda, 0xda, 0xae, + 0x73, 0xf4, 0x91, 0xeb, 0xe1, 0x0e, 0x9f, 0x02, 0x2f, 0xe1, 0x82, 0x0c, 0x7e, 0xdb, 0xb7, 0x43, + 0xec, 0xa1, 0xcf, 0xa8, 0xb7, 0x15, 0xf4, 0x8f, 0x6b, 0x95, 0xc0, 0x32, 0xb4, 0x22, 0xd7, 0xb7, + 0xd1, 0xf2, 0xe5, 0x18, 0x6a, 0x9a, 0x93, 0xe2, 0xf7, 0xd3, 0xc8, 0x78, 0x08, 0xab, 0x15, 0x7a, + 0x55, 0x66, 0x2f, 0xc1, 0xb4, 0x70, 0xcc, 0x0e, 0x7c, 0x86, 0x3e, 0x13, 0xba, 0xa7, 0xcd, 0x36, + 0xa7, 0x6d, 0x49, 0x92, 0xf1, 0x4f, 0x20, 0x52, 0xc7, 0x93, 0x20, 0xf6, 0xeb, 0xb5, 0xe6, 0x79, + 0x98, 0xcf, 0x88, 0xa8, 0xda, 0xb8, 0x07, 0x0b, 0x92, 0xfc, 0xc2, 0xef, 0xd5, 0xd6, 0xb5, 0x04, + 0xe7, 0x73, 0x42, 0x4a, 0xdb, 0xdd, 0xc4, 0x48, 0x76, 0x4f, 0x9c, 0xa8, 0x6c, 0x31, 0xf1, 0x20, + 0xbb, 0x2a, 0x8c, 0x5f, 0x35, 0x98, 0x4b, 0xc6, 0x48, 0xcd, 0xac, 0x9f, 0xb2, 0xec, 0x1a, 0x95, + 0x65, 0xd7, 0x1c, 0x94, 0x5d, 0x17, 0x66, 0xa3, 0x20, 0x0e, 0x6d, 0xb4, 0x1c, 0xca, 0xa8, 0xe5, + 0x07, 0x0e, 0xaa, 0xaa, 0x3c, 0x2b, 0xe9, 0x8f, 0x28, 0xa3, 0x4f, 0x03, 0x07, 0x8d, 0x0f, 0x92, + 0x8f, 0x92, 0xf9, 0x9a, 0xd7, 0x61, 0xce, 0xa3, 0x11, 0xb3, 0x68, 0xbf, 0x8f, 0xbe, 0x63, 0x51, + 0xc6, 0x4b, 0x42, 0x13, 0x25, 0x71, 0x96, 0x5f, 0x3c, 0x10, 0xf4, 0x07, 0xec, 0x69, 0x64, 0xfc, + 0xa8, 0xc1, 0x39, 0x2e, 0xcb, 0x4b, 0xb0, 0x56, 0xbc, 0xb3, 0xd0, 0xc0, 0x23, 0xa6, 0x02, 0xe5, + 0x47, 0xb2, 0x09, 0xf3, 0xaa, 0xd6, 0xdd, 0xc0, 0x1f, 0xb4, 0x41, 0x43, 0x08, 0x92, 0xc1, 0x55, + 0xda, 0x09, 0x17, 0xa1, 0x1d, 0xb1, 0xa0, 0x9f, 0x74, 0x55, 0x53, 0x76, 0x15, 0x27, 0xc9, 0xae, + 0x32, 0xfe, 0x05, 0xb3, 0x03, 0x9f, 0xea, 0x57, 0xe8, 0xb7, 0x5a, 0x32, 0x74, 0x9e, 0x53, 0xd7, + 0xdb, 0x41, 0xdf, 0xc1, 0xf0, 0x1d, 0x3b, 0x87, 0xdc, 0x81, 0x05, 0xd7, 0xf1, 0xd0, 0x62, 0x6e, + 0x0f, 0x83, 0x98, 0x59, 0x11, 0xda, 0x81, 0xef, 0x44, 0x49, 0x74, 0xfc, 0xee, 0xb9, 0xbc, 0xda, + 0x91, 0x37, 0xc6, 0x77, 0xe9, 0x04, 0x1b, 0xf6, 0x62, 0xb0, 0x87, 0x7d, 0x44, 0xae, 0x70, 0x1f, + 0xa9, 0x83, 0xa1, 0x0a, 0x63, 0x5a, 0x12, 0x3f, 0x11, 0x34, 0x9e, 0x1f, 0xc5, 0xb4, 0x1b, 0x38, + 0xc7, 0xc2, 0xa3, 0x69, 0x13, 0x24, 0xe9, 0x61, 0xe0, 0x1c, 0x8b, 0x51, 0x12, 0x59, 0xe2, 0x13, + 0xdb, 0xfb, 0xb1, 0x7f, 0x20, 0xbc, 0x69, 0x99, 0x6d, 0x37, 0xfa, 0x94, 0x46, 0x6c, 0x8b, 0x93, + 0x8c, 0xdf, 0x34, 0x58, 0x1e, 0xb8, 0x61, 0xa2, 0x8d, 0xee, 0xe1, 0xdf, 0x90, 0x0e, 0x2e, 0xa1, + 0x6a, 0x39, 0xf3, 0xe8, 0x52, 0xe5, 0x4e, 0xe4, 0x9d, 0x9a, 0xf8, 0xe2, 0x46, 0x2c, 0xf4, 0x12, + 0xc7, 0x55, 0x8b, 0xbe, 0x4a, 0x46, 0xd9, 0x63, 0x7b, 0x67, 0x9f, 0x86, 0x4e, 0xf4, 0x31, 0xfa, + 0x18, 0x52, 0xf6, 0x5e, 0xd6, 0xa4, 0xb1, 0x0e, 0x6b, 0x55, 0xda, 0x95, 0xfd, 0x9f, 0xd2, 0xbc, + 0x26, 0x2c, 0xef, 0x6d, 0x54, 0xac, 0x02, 0xa0, 0x6d, 0xb9, 0xbe, 0x83, 0x47, 0xc8, 0x53, 0xda, + 0xe8, 0xce, 0x98, 0x53, 0x68, 0x6f, 0x4b, 0xc2, 0x29, 0xa6, 0x42, 0x9a, 0xc1, 0xac, 0x8b, 0x2a, + 0x82, 0xef, 0x35, 0x58, 0xc9, 0x5e, 0xd7, 0x9f, 0x9c, 0xe4, 0x06, 0xcc, 0x45, 0xfb, 0x41, 0xec, + 0x39, 0x96, 0x23, 0x84, 0x2c, 0xb4, 0x8f, 0x44, 0x28, 0x2d, 0xf3, 0x9c, 0xbc, 0x90, 0xca, 0x1e, + 0xdb, 0x47, 0x23, 0xe2, 0x31, 0xd6, 0x92, 0x65, 0x97, 0x77, 0x43, 0xf9, 0x79, 0x94, 0x8f, 0xa2, + 0xf6, 0xe2, 0x79, 0xc7, 0x4c, 0x8b, 0x97, 0x5f, 0x99, 0x65, 0xe5, 0xd8, 0x17, 0x79, 0xc7, 0x4f, + 0xb1, 0xc7, 0x46, 0x99, 0xbe, 0x98, 0x2f, 0xef, 0xfc, 0xba, 0xfb, 0x1f, 0xac, 0x98, 0x48, 0x1d, + 0xc9, 0x24, 0x1e, 0x0e, 0xf5, 0x1f, 0x57, 0xbf, 0x9f, 0x81, 0x0b, 0xe5, 0xc2, 0x75, 0x1e, 0x58, + 0xff, 0x07, 0x3d, 0x7d, 0xc0, 0xf0, 0xfe, 0x8f, 0x18, 0xed, 0xf5, 0xd3, 0x09, 0x20, 0x07, 0xc5, + 0x92, 0x7a, 0xcd, 0x3c, 0x4f, 0xee, 0x93, 0x31, 0x50, 0x78, 0xfd, 0x34, 0x0a, 0xaf, 0x1f, 0x6e, + 0xc0, 0xa1, 0xac, 0xca, 0x80, 0x5c, 0x13, 0x4b, 0x0e, 0x65, 0x55, 0x06, 0x52, 0x61, 0x61, 0x60, + 0x5c, 0x1a, 0x50, 0xfc, 0xc2, 0xc0, 0x2a, 0x80, 0xda, 0x21, 0xb1, 0x9f, 0xbc, 0xe6, 0xa6, 0xe4, + 0x06, 0x89, 0xfd, 0xca, 0x45, 0x36, 0x59, 0xb9, 0xc8, 0xb2, 0x65, 0xd6, 0x2a, 0x4c, 0x93, 0xcf, + 0x00, 0x1e, 0xb9, 0xd1, 0x81, 0x4c, 0x32, 0xdf, 0x9c, 0x8e, 0x1b, 0x2a, 0x38, 0xc0, 0x8f, 0x9c, + 0x42, 0x3d, 0x4f, 0xa5, 0x8e, 0x1f, 0x39, 0x34, 0x8c, 0x23, 0x74, 0x54, 0x76, 0xc4, 0x99, 0xd3, + 0xf6, 0x42, 0x44, 0x95, 0x00, 0x71, 0x36, 0x7e, 0xd6, 0x60, 0xea, 0x09, 0xf6, 0x94, 0xe6, 0x35, + 0x80, 0xd7, 0x41, 0x18, 0xc4, 0xcc, 0xf5, 0x51, 0x2e, 0xfa, 0x71, 0x73, 0x88, 0xf2, 0xd7, 0xed, + 0x08, 0xa8, 0x8a, 0xde, 0x9e, 0x4a, 0xa6, 0x38, 0x73, 0xda, 0x3e, 0xd2, 0xbe, 0xca, 0x9f, 0x38, + 0x73, 0xa0, 0x1b, 0x31, 0x6a, 0x1f, 0x88, 0x64, 0x35, 0x4d, 0xf9, 0xe3, 0xee, 0x0f, 0xf3, 0x30, + 0x3d, 0x3c, 0xda, 0xc9, 0x2b, 0x68, 0x0f, 0x21, 0x74, 0x72, 0xa5, 0x08, 0xc4, 0x8b, 0x88, 0x5f, + 0xff, 0xc7, 0x08, 0x2e, 0xd5, 0x18, 0x63, 0xc4, 0x87, 0xb9, 0x02, 0x02, 0x26, 0x37, 0x8a, 0xd2, + 0x55, 0xf8, 0x5a, 0xbf, 0x59, 0x8b, 0x37, 0xb5, 0xc7, 0x60, 0xbe, 0x04, 0xd2, 0x92, 0x5b, 0x23, + 0xb4, 0x64, 0x60, 0xb5, 0x7e, 0xbb, 0x26, 0x77, 0x6a, 0xf5, 0x0d, 0x90, 0x22, 0xde, 0x25, 0x37, + 0x47, 0xaa, 0x19, 0xe0, 0x69, 0xfd, 0x56, 0x3d, 0xe6, 0xca, 0x40, 0x25, 0x12, 0x1e, 0x19, 0x68, + 0x06, 0x6b, 0x8f, 0x0c, 0x34, 0x07, 0xaf, 0xc7, 0xc8, 0x01, 0xcc, 0xe6, 0x51, 0x32, 0xb9, 0x5e, + 0xf5, 0xa7, 0x9b, 0x02, 0x08, 0xd7, 0x6f, 0xd4, 0x61, 0x4d, 0x8d, 0x21, 0x9c, 0xcd, 0x22, 0x59, + 0x72, 0xad, 0x28, 0x5f, 0x8a, 0xcb, 0xf5, 0xee, 0x68, 0xc6, 0xe1, 0x98, 0xf2, 0xe8, 0xb6, 0x2c, + 0xa6, 0x0a, 0xe8, 0x5c, 0x16, 0x53, 0x15, 0x58, 0x36, 0xc6, 0xc8, 0xd7, 0x09, 0x64, 0xca, 0xa1, + 0x3e, 0xb2, 0x51, 0xa5, 0xa6, 0x1c, 0x76, 0xea, 0x9b, 0xb5, 0xf9, 0x13, 0xdb, 0x77, 0x34, 0xde, + 0xeb, 0x43, 0xe0, 0xaf, 0xac, 0xd7, 0x8b, 0x70, 0xb2, 0xac, 0xd7, 0xcb, 0x10, 0xe4, 0x18, 0xd9, + 0x85, 0x99, 0x0c, 0x1c, 0x24, 0x57, 0xab, 0x24, 0xb3, 0xcb, 0x59, 0xbf, 0x36, 0x92, 0x2f, 0xb5, + 0x61, 0x25, 0xd3, 0x4b, 0x8d, 0xab, 0x4a, 0xe7, 0xb2, 0xf3, 0xea, 0xea, 0x28, 0xb6, 0xd4, 0xc0, + 0xe7, 0x00, 0x03, 0xf4, 0x46, 0x2e, 0x57, 0xc9, 0x0d, 0x7f, 0x8a, 0x2b, 0x27, 0x33, 0xa5, 0xaa, + 0xbf, 0x82, 0x85, 0xb2, 0x4d, 0x4f, 0x4a, 0xba, 0xf0, 0x84, 0xe7, 0x84, 0xbe, 0x51, 0x97, 0x3d, + 0x35, 0xfc, 0x02, 0x5a, 0x09, 0x76, 0x23, 0x97, 0x8a, 0xd2, 0x39, 0xac, 0xa9, 0x1b, 0x27, 0xb1, + 0x0c, 0x55, 0x53, 0x2f, 0x69, 0x9c, 0x01, 0xa8, 0xaa, 0x6e, 0x9c, 0x02, 0xfc, 0xab, 0x6e, 0x9c, + 0x22, 0x46, 0x13, 0xe6, 0xde, 0x24, 0xb8, 0x7a, 0x18, 0x83, 0x94, 0x0e, 0xd9, 0x2a, 0x88, 0x55, + 0x3a, 0x64, 0xab, 0x61, 0xcd, 0x18, 0xf9, 0x06, 0x16, 0xcb, 0xa1, 0x07, 0xa9, 0x6c, 0xbf, 0x0a, + 0x08, 0xa4, 0xdf, 0xa9, 0x2f, 0x90, 0x59, 0x2b, 0x05, 0xcc, 0x50, 0x1d, 0x71, 0x09, 0xf8, 0xa9, + 0x8e, 0xb8, 0x14, 0x86, 0x88, 0x1a, 0x2d, 0x03, 0x00, 0x65, 0x35, 0x7a, 0x02, 0x5e, 0xd1, 0x37, + 0xea, 0xb2, 0x67, 0xf6, 0x59, 0xf1, 0x7d, 0x4f, 0x46, 0xfa, 0x9f, 0x19, 0x55, 0xb7, 0x6b, 0x72, + 0xa7, 0x56, 0xdf, 0x26, 0xe3, 0x38, 0xf7, 0xb4, 0x27, 0x23, 0x03, 0xc8, 0x8d, 0xb0, 0xcd, 0xda, + 0xfc, 0x89, 0xed, 0xdd, 0x09, 0xf1, 0x4f, 0x96, 0x7b, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0xb2, + 0x60, 0x85, 0x8e, 0x7b, 0x19, 0x00, 0x00, } diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index 656d1eec9..f82b07e29 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -6,6 +6,7 @@ import ( "math" "os" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage" @@ -108,3 +109,41 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil } + +func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest) (*volume_server_pb.VolumeEcShardsMountResponse, error) { + + for _, shardId := range req.EcIndexes { + err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)) + + if err != nil { + glog.Errorf("ec shard mount %v: %v", req, err) + } else { + glog.V(2).Infof("ec shard mount %v", req) + } + + if err != nil { + return nil, fmt.Errorf("mount %d.%d: %v", req.VolumeId, shardId, err) + } + } + + return &volume_server_pb.VolumeEcShardsMountResponse{}, nil +} + +func (vs *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) { + + for _, shardId := range req.EcIndexes { + err := vs.store.UnmountEcShards(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)) + + if err != nil { + glog.Errorf("ec shard unmount %v: %v", req, err) + } else { + glog.V(2).Infof("ec shard unmount %v", req) + } + + if err != nil { + return nil, fmt.Errorf("unmount %d.%d: %v", req.VolumeId, shardId, err) + } + } + + return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil +} diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go index c9ec2fcd2..e0d028392 100644 --- a/weed/shell/command_ec_encode.go +++ b/weed/shell/command_ec_encode.go @@ -196,23 +196,36 @@ func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.Di targetServer *master_pb.DataNodeInfo, startFromShardId uint32, shardCount uint32, volumeId needle.VolumeId, collection string, existingLocation wdclient.Location) (copiedShardIds []uint32, err error) { - if targetServer.Id == existingLocation.Url { - return nil, nil - } - for shardId := startFromShardId; shardId < startFromShardId+shardCount; shardId++ { - fmt.Printf("copy %d.%d %s => %s\n", volumeId, shardId, existingLocation.Url, targetServer.Id) + fmt.Printf("allocate %d.%d %s => %s\n", volumeId, shardId, existingLocation.Url, targetServer.Id) copiedShardIds = append(copiedShardIds, shardId) } err = operation.WithVolumeServerClient(targetServer.Id, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, copyErr := volumeServerClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ - VolumeId: uint32(volumeId), - Collection: collection, - EcIndexes: copiedShardIds, - SourceDataNode: existingLocation.Url, + + if targetServer.Id != existingLocation.Url { + + _, copyErr := volumeServerClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(volumeId), + Collection: collection, + EcIndexes: copiedShardIds, + SourceDataNode: existingLocation.Url, + }) + if copyErr != nil { + return copyErr + } + } + + _, mountErr := volumeServerClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: uint32(volumeId), + Collection: collection, + EcIndexes: copiedShardIds, }) - return copyErr + if mountErr != nil { + return mountErr + } + + return nil }) if err != nil { diff --git a/weed/storage/disk_location_ec.go b/weed/storage/disk_location_ec.go index 3b2f1ec02..05acbb98b 100644 --- a/weed/storage/disk_location_ec.go +++ b/weed/storage/disk_location_ec.go @@ -16,26 +16,83 @@ var ( re = regexp.MustCompile("\\.ec[0-9][0-9]") ) -func (l *DiskLocation) loadEcShards(baseName string, shards []string, collection string, vid needle.VolumeId) (err error){ +func (l *DiskLocation) FindEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (*erasure_coding.EcVolumeShard, bool) { + l.ecShardsLock.RLock() + defer l.ecShardsLock.RUnlock() - for _, shard := range shards{ + ecShards, ok := l.ecShards[vid] + if !ok { + return nil, false + } + for _, ecShard := range ecShards { + if ecShard.ShardId == shardId { + return ecShard, true + } + } + return nil, false +} + +func (l *DiskLocation) LoadEcShard(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId) (err error) { + + ecVolumeShard, err := erasure_coding.NewEcVolumeShard(l.Directory, collection, vid, shardId) + if err != nil { + return fmt.Errorf("failed to create ec shard %d.%d: %v", vid, shardId, err) + } + l.ecShardsLock.Lock() + l.ecShards[vid] = append(l.ecShards[vid], ecVolumeShard) + l.ecShardsLock.Unlock() + + return nil +} + +func (l *DiskLocation) UnloadEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) bool { + + l.ecShardsLock.Lock() + defer l.ecShardsLock.Unlock() + + vidShards, found := l.ecShards[vid] + if !found { + return false + } + shardIndex := -1 + for i, shard := range vidShards { + if shard.ShardId == shardId { + shardIndex = i + break + } + } + if shardIndex < 0 { + return false + } + + if len(vidShards) == 1 { + delete(l.ecShards, vid) + return true + } + + l.ecShards[vid] = append(vidShards[:shardIndex], vidShards[shardIndex+1:]...) + + return true +} + +func (l *DiskLocation) loadEcShards(shards []string, collection string, vid needle.VolumeId) (err error) { + + for _, shard := range shards { shardId, err := strconv.ParseInt(path.Ext(shard)[3:], 10, 64) if err != nil { return fmt.Errorf("failed to parse ec shard name %v: %v", shard, err) } - ecVolumeShard, err := erasure_coding.NewEcVolumeShard(l.Directory, collection, vid, int(shardId)) + + err = l.LoadEcShard(collection, vid, erasure_coding.ShardId(shardId)) if err != nil { - return fmt.Errorf("failed to create ec shard %v: %v", shard, err) + return fmt.Errorf("failed to load ec shard %v: %v", shard, err) } - l.ecShardsLock.Lock() - l.ecShards[vid] = append(l.ecShards[vid], ecVolumeShard) - l.ecShardsLock.Unlock() } return nil } -func (l *DiskLocation) loadAllEcShards() (err error){ +func (l *DiskLocation) loadAllEcShards() (err error) { fileInfos, err := ioutil.ReadDir(l.Directory) if err != nil { @@ -48,8 +105,8 @@ func (l *DiskLocation) loadAllEcShards() (err error){ var sameVolumeShards []string var prevVolumeId needle.VolumeId - for _, fileInfo := range fileInfos{ - if fileInfo.IsDir(){ + for _, fileInfo := range fileInfos { + if fileInfo.IsDir() { continue } ext := path.Ext(fileInfo.Name()) @@ -61,18 +118,18 @@ func (l *DiskLocation) loadAllEcShards() (err error){ continue } - if re.MatchString(ext){ + if re.MatchString(ext) { if prevVolumeId == 0 || volumeId == prevVolumeId { sameVolumeShards = append(sameVolumeShards, fileInfo.Name()) - }else{ + } else { sameVolumeShards = []string{fileInfo.Name()} } prevVolumeId = volumeId continue } - if ext == ".ecx" && volumeId == prevVolumeId{ - if err = l.loadEcShards(baseName, sameVolumeShards, collection, volumeId);err!=nil{ + if ext == ".ecx" && volumeId == prevVolumeId { + if err = l.loadEcShards(sameVolumeShards, collection, volumeId); err != nil { return fmt.Errorf("loadEcShards collection:%v volumeId:%d : %v", collection, volumeId, err) } prevVolumeId = volumeId diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index f10f62135..11dcd0860 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -24,16 +24,16 @@ type EcVolumeShard struct { } type EcVolumeShards []*EcVolumeShard -func NewEcVolumeShard(dirname string, collection string, id needle.VolumeId, shardId int) (v *EcVolumeShard, e error) { +func NewEcVolumeShard(dirname string, collection string, id needle.VolumeId, shardId ShardId) (v *EcVolumeShard, e error) { - v = &EcVolumeShard{dir: dirname, Collection: collection, VolumeId: id, ShardId: ShardId(shardId)} + v = &EcVolumeShard{dir: dirname, Collection: collection, VolumeId: id, ShardId: shardId} baseFileName := v.FileName() if v.ecxFile, e = os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644); e != nil { return nil, fmt.Errorf("cannot read ec volume index %s.ecx: %v", baseFileName, e) } - if v.ecdFile, e = os.OpenFile(baseFileName+ToExt(shardId), os.O_RDONLY, 0644); e != nil { - return nil, fmt.Errorf("cannot read ec volume shard %s.%s: %v", baseFileName, ToExt(shardId), e) + if v.ecdFile, e = os.OpenFile(baseFileName+ToExt(int(shardId)), os.O_RDONLY, 0644); e != nil { + return nil, fmt.Errorf("cannot read ec volume shard %s.%s: %v", baseFileName, ToExt(int(shardId)), e) } return diff --git a/weed/storage/store.go b/weed/storage/store.go index d9c9b0e35..6aeced465 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -271,6 +271,7 @@ func (s *Store) UnmountVolume(i needle.VolumeId) error { Version: uint32(v.Version()), Ttl: v.Ttl.ToUint32(), } + for _, location := range s.Locations { if err := location.UnloadVolume(i); err == nil { glog.V(0).Infof("UnmountVolume %d", i) diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index 241adb24e..ad935609d 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -1,7 +1,12 @@ package storage import ( + "fmt" + + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" + "github.com/chrislusf/seaweedfs/weed/storage/needle" ) func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat { @@ -19,3 +24,56 @@ func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat { } } + +func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId) error { + for _, location := range s.Locations { + if err := location.LoadEcShard(collection, vid, shardId); err == nil { + glog.V(0).Infof("MountEcShards %d.%d", vid, shardId) + + var shardBits erasure_coding.ShardBits + + s.NewEcShardsChan <- master_pb.VolumeEcShardInformationMessage{ + Id: uint32(vid), + Collection: collection, + EcIndexBits: uint32(shardBits.AddShardId(shardId)), + } + return nil + } + } + + return fmt.Errorf("MountEcShards %d.%d not found on disk", vid, shardId) +} + +func (s *Store) UnmountEcShards(vid needle.VolumeId, shardId erasure_coding.ShardId) error { + + ecShard, found := s.findEcShard(vid, shardId) + if !found { + return nil + } + + var shardBits erasure_coding.ShardBits + message := master_pb.VolumeEcShardInformationMessage{ + Id: uint32(vid), + Collection: ecShard.Collection, + EcIndexBits: uint32(shardBits.AddShardId(shardId)), + } + + for _, location := range s.Locations { + if deleted := location.UnloadEcShard(vid, shardId); deleted { + glog.V(0).Infof("UnmountEcShards %d.%d", vid, shardId) + s.DeletedEcShardsChan <- message + return nil + } + } + + return fmt.Errorf("UnmountEcShards %d.%d not found on disk", vid, shardId) +} + +func (s *Store) findEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (*erasure_coding.EcVolumeShard, bool) { + for _, location := range s.Locations { + if v, found := location.FindEcShard(vid, shardId); found { + return v, found + } + } + return nil, false +}