From ec8de250e28b3356bb29b273baa557852f3c000b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Dec 2019 15:08:28 -0800 Subject: [PATCH] tiered storage: can copy to s3, read from s3 master not aware tiered volume yet, file assigning is not working yet --- weed/pb/volume_server.proto | 17 +- weed/pb/volume_server_pb/volume_server.pb.go | 534 +++++++++++------- .../volume_server_pb/volume_server_helper.go | 5 + weed/server/common.go | 7 +- weed/server/volume_grpc_tier.go | 75 ++- weed/shell/command_volume_tier.go | 68 ++- weed/storage/backend/backend.go | 27 +- weed/storage/backend/s3_backend/s3_backend.go | 56 +- weed/storage/backend/s3_backend/s3_upload.go | 66 ++- weed/storage/disk_location.go | 6 +- weed/storage/volume.go | 3 + weed/storage/volume_loading.go | 7 +- weed/storage/volume_tier.go | 104 ++++ 13 files changed, 689 insertions(+), 286 deletions(-) create mode 100644 weed/pb/volume_server_pb/volume_server_helper.go create mode 100644 weed/storage/volume_tier.go diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 94af4bea0..81c9901b9 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -68,7 +68,7 @@ service VolumeServer { } // tiered storage - rpc VolumeTierCopyDatToRemote (VolumeTierCopyDatToRemoteRequest) returns (VolumeTierCopyDatToRemoteResponse) { + rpc VolumeTierCopyDatToRemote (VolumeTierCopyDatToRemoteRequest) returns (stream VolumeTierCopyDatToRemoteResponse) { } // query @@ -329,18 +329,27 @@ message MemStatus { } // tired storage on volume servers -message TieredVolume { +message RemoteFile { string backend_type = 1; - string backend_name = 2; - uint64 version = 3; + string backend_id = 2; + string key = 3; + uint64 offset = 4; + uint64 file_size = 5; + uint64 modified_time = 6; +} +message VolumeTierInfo { + repeated RemoteFile files = 1; } message VolumeTierCopyDatToRemoteRequest { uint32 volume_id = 1; string collection = 2; string destination_backend_name = 3; + bool keep_local_dat_file = 4; } message VolumeTierCopyDatToRemoteResponse { + int64 processed = 1; + float processedPercentage = 2; } // select on volume servers diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 29d624209..030310644 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -65,7 +65,8 @@ It has these top-level messages: ReadVolumeFileStatusResponse DiskStatus MemStatus - TieredVolume + RemoteFile + VolumeTierInfo VolumeTierCopyDatToRemoteRequest VolumeTierCopyDatToRemoteResponse QueryRequest @@ -1398,49 +1399,90 @@ func (m *MemStatus) GetStack() uint64 { } // tired storage on volume servers -type TieredVolume struct { - BackendType string `protobuf:"bytes,1,opt,name=backend_type,json=backendType" json:"backend_type,omitempty"` - BackendName string `protobuf:"bytes,2,opt,name=backend_name,json=backendName" json:"backend_name,omitempty"` - Version uint64 `protobuf:"varint,3,opt,name=version" json:"version,omitempty"` +type RemoteFile struct { + BackendType string `protobuf:"bytes,1,opt,name=backend_type,json=backendType" json:"backend_type,omitempty"` + BackendId string `protobuf:"bytes,2,opt,name=backend_id,json=backendId" json:"backend_id,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key" json:"key,omitempty"` + Offset uint64 `protobuf:"varint,4,opt,name=offset" json:"offset,omitempty"` + FileSize uint64 `protobuf:"varint,5,opt,name=file_size,json=fileSize" json:"file_size,omitempty"` + ModifiedTime uint64 `protobuf:"varint,6,opt,name=modified_time,json=modifiedTime" json:"modified_time,omitempty"` } -func (m *TieredVolume) Reset() { *m = TieredVolume{} } -func (m *TieredVolume) String() string { return proto.CompactTextString(m) } -func (*TieredVolume) ProtoMessage() {} -func (*TieredVolume) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{56} } +func (m *RemoteFile) Reset() { *m = RemoteFile{} } +func (m *RemoteFile) String() string { return proto.CompactTextString(m) } +func (*RemoteFile) ProtoMessage() {} +func (*RemoteFile) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{56} } -func (m *TieredVolume) GetBackendType() string { +func (m *RemoteFile) GetBackendType() string { if m != nil { return m.BackendType } return "" } -func (m *TieredVolume) GetBackendName() string { +func (m *RemoteFile) GetBackendId() string { if m != nil { - return m.BackendName + return m.BackendId } return "" } -func (m *TieredVolume) GetVersion() uint64 { +func (m *RemoteFile) GetKey() string { if m != nil { - return m.Version + return m.Key + } + return "" +} + +func (m *RemoteFile) GetOffset() uint64 { + if m != nil { + return m.Offset } return 0 } +func (m *RemoteFile) GetFileSize() uint64 { + if m != nil { + return m.FileSize + } + return 0 +} + +func (m *RemoteFile) GetModifiedTime() uint64 { + if m != nil { + return m.ModifiedTime + } + return 0 +} + +type VolumeTierInfo struct { + Files []*RemoteFile `protobuf:"bytes,1,rep,name=files" json:"files,omitempty"` +} + +func (m *VolumeTierInfo) Reset() { *m = VolumeTierInfo{} } +func (m *VolumeTierInfo) String() string { return proto.CompactTextString(m) } +func (*VolumeTierInfo) ProtoMessage() {} +func (*VolumeTierInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} } + +func (m *VolumeTierInfo) GetFiles() []*RemoteFile { + if m != nil { + return m.Files + } + return nil +} + type VolumeTierCopyDatToRemoteRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` DestinationBackendName string `protobuf:"bytes,3,opt,name=destination_backend_name,json=destinationBackendName" json:"destination_backend_name,omitempty"` + KeepLocalDatFile bool `protobuf:"varint,4,opt,name=keep_local_dat_file,json=keepLocalDatFile" json:"keep_local_dat_file,omitempty"` } func (m *VolumeTierCopyDatToRemoteRequest) Reset() { *m = VolumeTierCopyDatToRemoteRequest{} } func (m *VolumeTierCopyDatToRemoteRequest) String() string { return proto.CompactTextString(m) } func (*VolumeTierCopyDatToRemoteRequest) ProtoMessage() {} func (*VolumeTierCopyDatToRemoteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{57} + return fileDescriptor0, []int{58} } func (m *VolumeTierCopyDatToRemoteRequest) GetVolumeId() uint32 { @@ -1464,14 +1506,37 @@ func (m *VolumeTierCopyDatToRemoteRequest) GetDestinationBackendName() string { return "" } +func (m *VolumeTierCopyDatToRemoteRequest) GetKeepLocalDatFile() bool { + if m != nil { + return m.KeepLocalDatFile + } + return false +} + type VolumeTierCopyDatToRemoteResponse struct { + Processed int64 `protobuf:"varint,1,opt,name=processed" json:"processed,omitempty"` + ProcessedPercentage float32 `protobuf:"fixed32,2,opt,name=processedPercentage" json:"processedPercentage,omitempty"` } func (m *VolumeTierCopyDatToRemoteResponse) Reset() { *m = VolumeTierCopyDatToRemoteResponse{} } func (m *VolumeTierCopyDatToRemoteResponse) String() string { return proto.CompactTextString(m) } func (*VolumeTierCopyDatToRemoteResponse) ProtoMessage() {} func (*VolumeTierCopyDatToRemoteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{58} + return fileDescriptor0, []int{59} +} + +func (m *VolumeTierCopyDatToRemoteResponse) GetProcessed() int64 { + if m != nil { + return m.Processed + } + return 0 +} + +func (m *VolumeTierCopyDatToRemoteResponse) GetProcessedPercentage() float32 { + if m != nil { + return m.ProcessedPercentage + } + return 0 } // select on volume servers @@ -1486,7 +1551,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{59} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{60} } func (m *QueryRequest) GetSelections() []string { if m != nil { @@ -1532,7 +1597,7 @@ type QueryRequest_Filter struct { func (m *QueryRequest_Filter) Reset() { *m = QueryRequest_Filter{} } func (m *QueryRequest_Filter) String() string { return proto.CompactTextString(m) } func (*QueryRequest_Filter) ProtoMessage() {} -func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{59, 0} } +func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{60, 0} } func (m *QueryRequest_Filter) GetField() string { if m != nil { @@ -1567,7 +1632,7 @@ func (m *QueryRequest_InputSerialization) Reset() { *m = QueryRequest_In func (m *QueryRequest_InputSerialization) String() string { return proto.CompactTextString(m) } func (*QueryRequest_InputSerialization) ProtoMessage() {} func (*QueryRequest_InputSerialization) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{59, 1} + return fileDescriptor0, []int{60, 1} } func (m *QueryRequest_InputSerialization) GetCompressionType() string { @@ -1615,7 +1680,7 @@ func (m *QueryRequest_InputSerialization_CSVInput) Reset() { func (m *QueryRequest_InputSerialization_CSVInput) String() string { return proto.CompactTextString(m) } func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_CSVInput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{59, 1, 0} + return fileDescriptor0, []int{60, 1, 0} } func (m *QueryRequest_InputSerialization_CSVInput) GetFileHeaderInfo() string { @@ -1677,7 +1742,7 @@ func (m *QueryRequest_InputSerialization_JSONInput) Reset() { func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) } func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{59, 1, 1} + return fileDescriptor0, []int{60, 1, 1} } func (m *QueryRequest_InputSerialization_JSONInput) GetType() string { @@ -1698,7 +1763,7 @@ func (m *QueryRequest_InputSerialization_ParquetInput) String() string { } func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_ParquetInput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{59, 1, 2} + return fileDescriptor0, []int{60, 1, 2} } type QueryRequest_OutputSerialization struct { @@ -1710,7 +1775,7 @@ func (m *QueryRequest_OutputSerialization) Reset() { *m = QueryRequest_O func (m *QueryRequest_OutputSerialization) String() string { return proto.CompactTextString(m) } func (*QueryRequest_OutputSerialization) ProtoMessage() {} func (*QueryRequest_OutputSerialization) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{59, 2} + return fileDescriptor0, []int{60, 2} } func (m *QueryRequest_OutputSerialization) GetCsvOutput() *QueryRequest_OutputSerialization_CSVOutput { @@ -1743,7 +1808,7 @@ func (m *QueryRequest_OutputSerialization_CSVOutput) String() string { } func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {} func (*QueryRequest_OutputSerialization_CSVOutput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{59, 2, 0} + return fileDescriptor0, []int{60, 2, 0} } func (m *QueryRequest_OutputSerialization_CSVOutput) GetQuoteFields() string { @@ -1793,7 +1858,7 @@ func (m *QueryRequest_OutputSerialization_JSONOutput) String() string { } func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {} func (*QueryRequest_OutputSerialization_JSONOutput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{59, 2, 1} + return fileDescriptor0, []int{60, 2, 1} } func (m *QueryRequest_OutputSerialization_JSONOutput) GetRecordDelimiter() string { @@ -1810,7 +1875,7 @@ type QueriedStripe struct { func (m *QueriedStripe) Reset() { *m = QueriedStripe{} } func (m *QueriedStripe) String() string { return proto.CompactTextString(m) } func (*QueriedStripe) ProtoMessage() {} -func (*QueriedStripe) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{60} } +func (*QueriedStripe) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{61} } func (m *QueriedStripe) GetRecords() []byte { if m != nil { @@ -1876,7 +1941,8 @@ func init() { proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse") proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus") proto.RegisterType((*MemStatus)(nil), "volume_server_pb.MemStatus") - proto.RegisterType((*TieredVolume)(nil), "volume_server_pb.TieredVolume") + proto.RegisterType((*RemoteFile)(nil), "volume_server_pb.RemoteFile") + proto.RegisterType((*VolumeTierInfo)(nil), "volume_server_pb.VolumeTierInfo") proto.RegisterType((*VolumeTierCopyDatToRemoteRequest)(nil), "volume_server_pb.VolumeTierCopyDatToRemoteRequest") proto.RegisterType((*VolumeTierCopyDatToRemoteResponse)(nil), "volume_server_pb.VolumeTierCopyDatToRemoteResponse") proto.RegisterType((*QueryRequest)(nil), "volume_server_pb.QueryRequest") @@ -1932,7 +1998,7 @@ type VolumeServerClient interface { VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) VolumeEcBlobDelete(ctx context.Context, in *VolumeEcBlobDeleteRequest, opts ...grpc.CallOption) (*VolumeEcBlobDeleteResponse, error) // tiered storage - VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (*VolumeTierCopyDatToRemoteResponse, error) + VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierCopyDatToRemoteClient, error) // query Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) } @@ -2271,17 +2337,40 @@ func (c *volumeServerClient) VolumeEcBlobDelete(ctx context.Context, in *VolumeE return out, nil } -func (c *volumeServerClient) VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (*VolumeTierCopyDatToRemoteResponse, error) { - out := new(VolumeTierCopyDatToRemoteResponse) - err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote", in, out, c.cc, opts...) +func (c *volumeServerClient) VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTierCopyDatToRemoteClient, error) { + stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[4], c.cc, "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote", opts...) if err != nil { return nil, err } - return out, nil + x := &volumeServerVolumeTierCopyDatToRemoteClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeTierCopyDatToRemoteClient interface { + Recv() (*VolumeTierCopyDatToRemoteResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeTierCopyDatToRemoteClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeTierCopyDatToRemoteClient) Recv() (*VolumeTierCopyDatToRemoteResponse, error) { + m := new(VolumeTierCopyDatToRemoteResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil } func (c *volumeServerClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) { - stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[4], c.cc, "/volume_server_pb.VolumeServer/Query", opts...) + stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[5], c.cc, "/volume_server_pb.VolumeServer/Query", opts...) if err != nil { return nil, err } @@ -2345,7 +2434,7 @@ type VolumeServerServer interface { VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) // tiered storage - VolumeTierCopyDatToRemote(context.Context, *VolumeTierCopyDatToRemoteRequest) (*VolumeTierCopyDatToRemoteResponse, error) + VolumeTierCopyDatToRemote(*VolumeTierCopyDatToRemoteRequest, VolumeServer_VolumeTierCopyDatToRemoteServer) error // query Query(*QueryRequest, VolumeServer_QueryServer) error } @@ -2834,22 +2923,25 @@ func _VolumeServer_VolumeEcBlobDelete_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } -func _VolumeServer_VolumeTierCopyDatToRemote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(VolumeTierCopyDatToRemoteRequest) - if err := dec(in); err != nil { - return nil, err +func _VolumeServer_VolumeTierCopyDatToRemote_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeTierCopyDatToRemoteRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - if interceptor == nil { - return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(ctx, req.(*VolumeTierCopyDatToRemoteRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(m, &volumeServerVolumeTierCopyDatToRemoteServer{stream}) +} + +type VolumeServer_VolumeTierCopyDatToRemoteServer interface { + Send(*VolumeTierCopyDatToRemoteResponse) error + grpc.ServerStream +} + +type volumeServerVolumeTierCopyDatToRemoteServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeTierCopyDatToRemoteServer) Send(m *VolumeTierCopyDatToRemoteResponse) error { + return x.ServerStream.SendMsg(m) } func _VolumeServer_Query_Handler(srv interface{}, stream grpc.ServerStream) error { @@ -2965,10 +3057,6 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ MethodName: "VolumeEcBlobDelete", Handler: _VolumeServer_VolumeEcBlobDelete_Handler, }, - { - MethodName: "VolumeTierCopyDatToRemote", - Handler: _VolumeServer_VolumeTierCopyDatToRemote_Handler, - }, }, Streams: []grpc.StreamDesc{ { @@ -2991,6 +3079,11 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ Handler: _VolumeServer_VolumeEcShardRead_Handler, ServerStreams: true, }, + { + StreamName: "VolumeTierCopyDatToRemote", + Handler: _VolumeServer_VolumeTierCopyDatToRemote_Handler, + ServerStreams: true, + }, { StreamName: "Query", Handler: _VolumeServer_Query_Handler, @@ -3003,169 +3096,176 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 2613 bytes of a gzipped FileDescriptorProto + // 2735 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x3a, 0xcd, 0x73, 0x1c, 0x47, - 0xf5, 0x5a, 0xaf, 0x3e, 0x76, 0xdf, 0xae, 0x2c, 0xb9, 0x25, 0x4b, 0xeb, 0xb1, 0x25, 0xcb, 0xe3, - 0x24, 0x96, 0xed, 0x44, 0x76, 0xe4, 0xdf, 0x8f, 0x98, 0x84, 0x00, 0xb6, 0x6c, 0x83, 0x49, 0x2c, - 0x93, 0x91, 0x62, 0x02, 0x4e, 0x31, 0xd5, 0x9a, 0x69, 0x59, 0x83, 0x66, 0xa6, 0xc7, 0x33, 0x3d, - 0xb2, 0xd7, 0x05, 0xa7, 0x50, 0xdc, 0xf8, 0x03, 0x72, 0xe6, 0xce, 0x81, 0x0b, 0x7f, 0x00, 0x17, - 0xfe, 0x00, 0xb8, 0x72, 0xe1, 0xcc, 0x81, 0x1b, 0x55, 0x5c, 0xa8, 0xfe, 0x98, 0xd9, 0xf9, 0xd4, - 0x8e, 0x62, 0x57, 0x51, 0xdc, 0x66, 0x5f, 0xbf, 0xef, 0x7e, 0xaf, 0xfb, 0xf5, 0x7b, 0x0b, 0x0b, - 0x47, 0xd4, 0x8d, 0x3d, 0x62, 0x46, 0x24, 0x3c, 0x22, 0xe1, 0x46, 0x10, 0x52, 0x46, 0xd1, 0x7c, - 0x0e, 0x68, 0x06, 0x7b, 0xfa, 0x0d, 0x40, 0x77, 0x31, 0xb3, 0x0e, 0xee, 0x11, 0x97, 0x30, 0x62, - 0x90, 0xe7, 0x31, 0x89, 0x18, 0x3a, 0x07, 0x9d, 0x7d, 0xc7, 0x25, 0xa6, 0x63, 0x47, 0x83, 0xd6, - 0x5a, 0x7b, 0xbd, 0x6b, 0xcc, 0xf0, 0xdf, 0x0f, 0xed, 0x48, 0x7f, 0x0c, 0x0b, 0x39, 0x82, 0x28, - 0xa0, 0x7e, 0x44, 0xd0, 0x6d, 0x98, 0x09, 0x49, 0x14, 0xbb, 0x4c, 0x12, 0xf4, 0x36, 0x57, 0x37, - 0x8a, 0xb2, 0x36, 0x52, 0x92, 0xd8, 0x65, 0x46, 0x82, 0xae, 0x7f, 0xd5, 0x82, 0x7e, 0x76, 0x05, - 0x2d, 0xc3, 0x8c, 0x12, 0x3e, 0x68, 0xad, 0xb5, 0xd6, 0xbb, 0xc6, 0xb4, 0x94, 0x8d, 0x96, 0x60, - 0x3a, 0x62, 0x98, 0xc5, 0xd1, 0xe0, 0xd4, 0x5a, 0x6b, 0x7d, 0xca, 0x50, 0xbf, 0xd0, 0x22, 0x4c, - 0x91, 0x30, 0xa4, 0xe1, 0xa0, 0x2d, 0xd0, 0xe5, 0x0f, 0x84, 0x60, 0x32, 0x72, 0x5e, 0x91, 0xc1, - 0xe4, 0x5a, 0x6b, 0x7d, 0xd6, 0x10, 0xdf, 0x68, 0x00, 0x33, 0x47, 0x24, 0x8c, 0x1c, 0xea, 0x0f, - 0xa6, 0x04, 0x38, 0xf9, 0xa9, 0xcf, 0xc0, 0xd4, 0x7d, 0x2f, 0x60, 0x43, 0xfd, 0x03, 0x18, 0x3c, - 0xc1, 0x56, 0x1c, 0x7b, 0x4f, 0x84, 0xfa, 0x5b, 0x07, 0xc4, 0x3a, 0x4c, 0xdc, 0x72, 0x1e, 0xba, - 0xca, 0x28, 0xa5, 0xdb, 0xac, 0xd1, 0x91, 0x80, 0x87, 0xb6, 0xfe, 0x7d, 0x38, 0x57, 0x41, 0xa8, - 0xdc, 0x73, 0x19, 0x66, 0x9f, 0xe1, 0x70, 0x0f, 0x3f, 0x23, 0x66, 0x88, 0x99, 0x43, 0x05, 0x75, - 0xcb, 0xe8, 0x2b, 0xa0, 0xc1, 0x61, 0xfa, 0x53, 0xd0, 0x72, 0x1c, 0xa8, 0x17, 0x60, 0x8b, 0x35, - 0x11, 0x8e, 0xd6, 0xa0, 0x17, 0x84, 0x04, 0xbb, 0x2e, 0xb5, 0x30, 0x23, 0xc2, 0x3f, 0x6d, 0x23, - 0x0b, 0xd2, 0x57, 0xe0, 0x7c, 0x25, 0x73, 0xa9, 0xa0, 0x7e, 0xbb, 0xa0, 0x3d, 0xf5, 0x3c, 0xa7, - 0x91, 0x68, 0xfd, 0x42, 0x49, 0x6b, 0x41, 0xa9, 0xf8, 0x7e, 0xbb, 0xb0, 0xea, 0x12, 0xec, 0xc7, - 0x41, 0x23, 0xc6, 0x45, 0x8d, 0x13, 0xd2, 0x94, 0xf3, 0xb2, 0x0c, 0x9b, 0x2d, 0xea, 0xba, 0xc4, - 0x62, 0x0e, 0xf5, 0x13, 0xb6, 0xab, 0x00, 0x56, 0x0a, 0x54, 0x41, 0x94, 0x81, 0xe8, 0x1a, 0x0c, - 0xca, 0xa4, 0x8a, 0xed, 0xdf, 0x5a, 0x70, 0xf6, 0x8e, 0x72, 0x9a, 0x14, 0xdc, 0x68, 0x03, 0xf2, - 0x22, 0x4f, 0x15, 0x45, 0x16, 0x37, 0xa8, 0x5d, 0xda, 0x20, 0x8e, 0x11, 0x92, 0xc0, 0x75, 0x2c, - 0x2c, 0x58, 0x4c, 0x0a, 0x16, 0x59, 0x10, 0x9a, 0x87, 0x36, 0x63, 0xae, 0x88, 0xdc, 0xae, 0xc1, - 0x3f, 0xd1, 0x26, 0x2c, 0x79, 0xc4, 0xa3, 0xe1, 0xd0, 0xf4, 0x70, 0x60, 0x7a, 0xf8, 0xa5, 0xc9, - 0xc3, 0xdc, 0xf4, 0xf6, 0x06, 0xd3, 0x42, 0x3f, 0x24, 0x57, 0x1f, 0xe1, 0xe0, 0x11, 0x7e, 0xb9, - 0xe3, 0xbc, 0x22, 0x8f, 0xf6, 0xf4, 0x01, 0x2c, 0x15, 0xed, 0x53, 0xa6, 0x7f, 0x0b, 0x96, 0x25, - 0x64, 0x67, 0xe8, 0x5b, 0x3b, 0x22, 0xb7, 0x1a, 0x6d, 0xd4, 0xbf, 0x5b, 0x30, 0x28, 0x13, 0xaa, - 0xc8, 0x7f, 0x5d, 0xaf, 0x9d, 0xd8, 0x27, 0x17, 0xa1, 0xc7, 0xb0, 0xe3, 0x9a, 0x74, 0x7f, 0x3f, - 0x22, 0x4c, 0x38, 0x62, 0xd2, 0x00, 0x0e, 0x7a, 0x2c, 0x20, 0xe8, 0x2a, 0xcc, 0x5b, 0x32, 0xfa, - 0xcd, 0x90, 0x1c, 0x39, 0xe2, 0x34, 0x98, 0x11, 0x8a, 0xcd, 0x59, 0x49, 0x56, 0x48, 0x30, 0xd2, - 0x61, 0xd6, 0xb1, 0x5f, 0x9a, 0xe2, 0x38, 0x12, 0x87, 0x49, 0x47, 0x70, 0xeb, 0x39, 0xf6, 0xcb, - 0x07, 0x8e, 0x4b, 0xb8, 0x47, 0xf5, 0x27, 0x70, 0x41, 0x1a, 0xff, 0xd0, 0xb7, 0x42, 0xe2, 0x11, - 0x9f, 0x61, 0x77, 0x8b, 0x06, 0xc3, 0x46, 0x61, 0x73, 0x0e, 0x3a, 0x91, 0xe3, 0x5b, 0xc4, 0xf4, - 0xe5, 0xa1, 0x36, 0x69, 0xcc, 0x88, 0xdf, 0xdb, 0x91, 0x7e, 0x17, 0x56, 0x6a, 0xf8, 0x2a, 0xcf, - 0x5e, 0x82, 0xbe, 0x50, 0xcc, 0xa2, 0x3e, 0x23, 0x3e, 0x13, 0xbc, 0xfb, 0x46, 0x8f, 0xc3, 0xb6, - 0x24, 0x48, 0x7f, 0x1f, 0x90, 0xe4, 0xf1, 0x88, 0xc6, 0x7e, 0xb3, 0x74, 0x3e, 0x0b, 0x0b, 0x39, - 0x12, 0x15, 0x1b, 0xb7, 0x60, 0x51, 0x82, 0x3f, 0xf7, 0xbd, 0xc6, 0xbc, 0x96, 0xe1, 0x6c, 0x81, - 0x48, 0x71, 0xdb, 0x4c, 0x84, 0xe4, 0xaf, 0x9d, 0x63, 0x99, 0x2d, 0x25, 0x1a, 0xe4, 0x6f, 0x1e, - 0x71, 0x72, 0x49, 0x85, 0x71, 0x78, 0x68, 0x10, 0x6c, 0x53, 0xdf, 0x1d, 0x36, 0x3e, 0xb9, 0x2a, - 0x28, 0x15, 0xdf, 0xdf, 0xb7, 0xe0, 0x4c, 0x72, 0xa4, 0x35, 0xdc, 0xcd, 0x13, 0x86, 0x73, 0xbb, - 0x36, 0x9c, 0x27, 0x47, 0xe1, 0xbc, 0x0e, 0xf3, 0x11, 0x8d, 0x43, 0x8b, 0x98, 0x36, 0x66, 0xd8, - 0xf4, 0xa9, 0x4d, 0x54, 0xb4, 0x9f, 0x96, 0xf0, 0x7b, 0x98, 0xe1, 0x6d, 0x6a, 0x13, 0xfd, 0x7b, - 0xc9, 0x66, 0xe7, 0xa2, 0xe4, 0x2a, 0x9c, 0x71, 0x71, 0xc4, 0x4c, 0x1c, 0x04, 0xc4, 0xb7, 0x4d, - 0xcc, 0x78, 0xa8, 0xb5, 0x44, 0xa8, 0x9d, 0xe6, 0x0b, 0x77, 0x04, 0xfc, 0x0e, 0xdb, 0x8e, 0xf4, - 0xbf, 0xb4, 0x60, 0x8e, 0xd3, 0xf2, 0xd0, 0x6e, 0x64, 0xef, 0x3c, 0xb4, 0xc9, 0x4b, 0xa6, 0x0c, - 0xe5, 0x9f, 0xe8, 0x06, 0x2c, 0xa8, 0x1c, 0x72, 0xa8, 0x3f, 0x4a, 0xaf, 0xb6, 0x3c, 0x8d, 0x46, - 0x4b, 0x69, 0x86, 0x5d, 0x84, 0x5e, 0xc4, 0x68, 0x90, 0x64, 0xeb, 0xa4, 0xcc, 0x56, 0x0e, 0x52, - 0xd9, 0x9a, 0xf7, 0xe9, 0x54, 0x85, 0x4f, 0xfb, 0x4e, 0x64, 0x12, 0xcb, 0x94, 0x5a, 0x89, 0x7c, - 0xef, 0x18, 0xe0, 0x44, 0xf7, 0x2d, 0xe9, 0x0d, 0xfd, 0xff, 0x61, 0x7e, 0x64, 0x55, 0xf3, 0xdc, - 0xf9, 0xaa, 0x95, 0x1c, 0x87, 0xbb, 0xd8, 0x71, 0x77, 0x88, 0x6f, 0x93, 0xf0, 0x35, 0x73, 0x1a, - 0xdd, 0x84, 0x45, 0xc7, 0x76, 0x89, 0xc9, 0x1c, 0x8f, 0xd0, 0x98, 0x99, 0x11, 0xb1, 0xa8, 0x6f, - 0x47, 0x89, 0x7f, 0xf8, 0xda, 0xae, 0x5c, 0xda, 0x91, 0x2b, 0xfa, 0xaf, 0xd3, 0xb3, 0x35, 0xab, - 0xc5, 0xa8, 0xaa, 0xf0, 0x09, 0xe1, 0x0c, 0x0f, 0x08, 0xb6, 0x49, 0xa8, 0xcc, 0xe8, 0x4b, 0xe0, - 0x0f, 0x05, 0x8c, 0x7b, 0x58, 0x21, 0xed, 0x51, 0x7b, 0x28, 0x34, 0xea, 0x1b, 0x20, 0x41, 0x77, - 0xa9, 0x3d, 0x14, 0x87, 0x5c, 0x64, 0x8a, 0x20, 0xb1, 0x0e, 0x62, 0xff, 0x50, 0x68, 0xd3, 0x31, - 0x7a, 0x4e, 0xf4, 0x29, 0x8e, 0xd8, 0x16, 0x07, 0xe9, 0x7f, 0x6c, 0x25, 0x59, 0xc6, 0xd5, 0x30, - 0x88, 0x45, 0x9c, 0xa3, 0xff, 0x82, 0x3b, 0x38, 0x85, 0xca, 0x86, 0x5c, 0x75, 0xa9, 0x12, 0x06, - 0xc9, 0x35, 0x75, 0x17, 0x89, 0x95, 0x51, 0x92, 0xe7, 0x15, 0x57, 0x49, 0xfe, 0x65, 0x72, 0xc8, - 0xde, 0xb7, 0x76, 0x0e, 0x70, 0x68, 0x47, 0x3f, 0x20, 0x3e, 0x09, 0x31, 0x7b, 0x23, 0x97, 0xbe, - 0xbe, 0x06, 0xab, 0x75, 0xdc, 0x95, 0xfc, 0xa7, 0xc9, 0xe5, 0x91, 0x60, 0x18, 0x64, 0x2f, 0x76, - 0x5c, 0xfb, 0x8d, 0x88, 0xff, 0xa4, 0x68, 0x5c, 0xca, 0x5c, 0xc5, 0xcf, 0x35, 0x38, 0x13, 0x0a, - 0x10, 0x33, 0x23, 0x8e, 0x90, 0xd6, 0xfb, 0xb3, 0xc6, 0x9c, 0x5a, 0x10, 0x84, 0xbc, 0xee, 0xff, - 0x53, 0x1a, 0x01, 0x09, 0xb7, 0x37, 0x76, 0x2c, 0x9e, 0x87, 0xee, 0x48, 0x7c, 0x5b, 0x88, 0xef, - 0x44, 0x4a, 0x2e, 0x8f, 0x4e, 0x8b, 0x06, 0x43, 0x93, 0x58, 0xf2, 0x1e, 0x16, 0x5b, 0xdd, 0x31, - 0x7a, 0x1c, 0x78, 0xdf, 0x12, 0xd7, 0xf0, 0x09, 0xce, 0xc8, 0x34, 0x1a, 0xf2, 0x46, 0xa8, 0xdd, - 0x78, 0x01, 0xe7, 0xf3, 0xab, 0xcd, 0xaf, 0xa7, 0xd7, 0x32, 0x52, 0x5f, 0x2d, 0x86, 0x41, 0xe1, - 0x8e, 0x3b, 0x2a, 0xaa, 0xdd, 0xf8, 0x3e, 0x7f, 0x3d, 0xbd, 0x56, 0x8a, 0x0e, 0xc9, 0x17, 0x05, - 0x5f, 0x14, 0xd5, 0x3e, 0x41, 0x71, 0x70, 0xbc, 0xe0, 0x8b, 0xc5, 0xd0, 0x2d, 0x56, 0x10, 0x5f, - 0xa7, 0xe7, 0xa2, 0xc2, 0xe0, 0xf7, 0x77, 0xe3, 0xf3, 0x48, 0xc9, 0x15, 0xee, 0x98, 0x35, 0x66, - 0x94, 0x58, 0xfe, 0xc0, 0x54, 0xf7, 0x90, 0xac, 0xcf, 0xd5, 0xaf, 0xdc, 0x53, 0xb2, 0xad, 0x9e, - 0x92, 0xc9, 0x13, 0xf9, 0x90, 0x0c, 0x45, 0xac, 0x4d, 0xca, 0x27, 0xf2, 0x27, 0x64, 0xa8, 0x6f, - 0x17, 0x32, 0x45, 0xaa, 0xa6, 0x72, 0x0e, 0xc1, 0x24, 0x0f, 0x52, 0x75, 0x54, 0x8b, 0x6f, 0xb4, - 0x02, 0xe0, 0x44, 0xa6, 0x2d, 0xf6, 0x5c, 0x2a, 0xd5, 0x31, 0xba, 0x8e, 0x0a, 0x02, 0x5b, 0xff, - 0x6d, 0x26, 0xf5, 0xee, 0xba, 0x74, 0xef, 0x0d, 0x46, 0x65, 0xd6, 0x8a, 0x76, 0xce, 0x8a, 0xec, - 0x5b, 0x79, 0x32, 0xff, 0x56, 0xce, 0x24, 0x51, 0x56, 0x1d, 0xb5, 0x33, 0x1f, 0xc2, 0x79, 0x6e, - 0xb0, 0xc4, 0x10, 0x55, 0x72, 0xf3, 0x97, 0xc4, 0x3f, 0x4e, 0xc1, 0x85, 0x6a, 0xe2, 0x26, 0xaf, - 0x89, 0x8f, 0x40, 0x4b, 0xab, 0x75, 0x7e, 0xa5, 0x44, 0x0c, 0x7b, 0x41, 0x7a, 0xa9, 0xc8, 0xbb, - 0x67, 0x59, 0x95, 0xee, 0xbb, 0xc9, 0x7a, 0x72, 0xb3, 0x94, 0x4a, 0xfd, 0x76, 0xa9, 0xd4, 0xe7, - 0x02, 0x6c, 0xcc, 0xea, 0x04, 0xc8, 0xda, 0x65, 0xd9, 0xc6, 0xac, 0x4e, 0x40, 0x4a, 0x2c, 0x04, - 0xc8, 0xa8, 0xe9, 0x29, 0x7c, 0x21, 0x60, 0x05, 0x40, 0x95, 0x25, 0xb1, 0x9f, 0x3c, 0x5d, 0xba, - 0xb2, 0x28, 0x89, 0xfd, 0xda, 0xea, 0x6a, 0xa6, 0xb6, 0xba, 0xca, 0x6f, 0x7f, 0xa7, 0x74, 0x43, - 0x7c, 0x01, 0x70, 0xcf, 0x89, 0x0e, 0xa5, 0x93, 0x79, 0x39, 0x67, 0x3b, 0xa1, 0x7a, 0x2f, 0xf3, - 0x4f, 0x0e, 0xc1, 0xae, 0xab, 0x5c, 0xc7, 0x3f, 0x79, 0xf8, 0xc6, 0x11, 0xb1, 0x95, 0x77, 0xc4, - 0x37, 0x87, 0xed, 0x87, 0x84, 0x28, 0x07, 0x88, 0x6f, 0xfd, 0x77, 0x2d, 0xe8, 0x3e, 0x22, 0x9e, - 0xe2, 0xbc, 0x0a, 0xf0, 0x8c, 0x86, 0x34, 0x66, 0x8e, 0x4f, 0x64, 0xf5, 0x39, 0x65, 0x64, 0x20, - 0xdf, 0x5c, 0x8e, 0x48, 0x4d, 0xe2, 0xee, 0x2b, 0x67, 0x8a, 0x6f, 0x0e, 0x3b, 0x20, 0x38, 0x50, - 0xfe, 0x13, 0xdf, 0x68, 0x11, 0xa6, 0x22, 0x86, 0xad, 0x43, 0xe1, 0xac, 0x49, 0x43, 0xfe, 0xd0, - 0x7d, 0xe8, 0xef, 0x3a, 0x24, 0x24, 0x2a, 0xe0, 0x78, 0x59, 0xb8, 0x87, 0xad, 0x43, 0x5e, 0x28, - 0xb3, 0x61, 0x40, 0x94, 0x2b, 0x7a, 0x0a, 0xb6, 0x3b, 0x0c, 0x72, 0x28, 0x3e, 0xf6, 0x88, 0xca, - 0xa9, 0x04, 0x65, 0x1b, 0x7b, 0xb9, 0x2e, 0x93, 0xca, 0xa9, 0x24, 0x73, 0xbe, 0x6e, 0xc1, 0x9a, - 0xaa, 0x46, 0x1c, 0x12, 0xf2, 0xbb, 0xe7, 0x1e, 0x66, 0xbb, 0xd4, 0x20, 0x1e, 0x7d, 0x43, 0x09, - 0x7d, 0x1b, 0x06, 0x36, 0x89, 0x98, 0xe3, 0x8b, 0xf7, 0x84, 0x99, 0x53, 0x55, 0xbe, 0x37, 0x96, - 0x32, 0xeb, 0x77, 0x47, 0x5a, 0xeb, 0x97, 0xe1, 0xd2, 0x31, 0xaa, 0xa9, 0xe4, 0xfe, 0x57, 0x1f, - 0xfa, 0x9f, 0xc5, 0x24, 0x1c, 0x66, 0x5a, 0x2d, 0x11, 0x51, 0xc2, 0x93, 0x5e, 0x61, 0x06, 0xc2, - 0xa3, 0x7e, 0x3f, 0xa4, 0x9e, 0x99, 0xb6, 0x13, 0x4f, 0x09, 0x94, 0x1e, 0x07, 0x3e, 0x90, 0x2d, - 0x45, 0xf4, 0x31, 0x4c, 0xef, 0x3b, 0x2e, 0x23, 0xb2, 0x81, 0xd7, 0xdb, 0x7c, 0xbb, 0xdc, 0x3a, - 0xcc, 0xca, 0xdc, 0x78, 0x20, 0x90, 0x0d, 0x45, 0x84, 0xf6, 0x60, 0xc1, 0xf1, 0x03, 0x51, 0x3e, - 0x86, 0x0e, 0x76, 0x9d, 0x57, 0xa3, 0x66, 0x41, 0x6f, 0xf3, 0xfd, 0x31, 0xbc, 0x1e, 0x72, 0xca, - 0x9d, 0x2c, 0xa1, 0x81, 0x9c, 0x12, 0x0c, 0x11, 0x58, 0xa4, 0x31, 0x2b, 0x0b, 0x99, 0x12, 0x42, - 0x36, 0xc7, 0x08, 0x79, 0x2c, 0x48, 0xf3, 0x52, 0x16, 0x68, 0x19, 0xa8, 0x6d, 0xc3, 0xb4, 0x34, - 0x8e, 0xc7, 0xeb, 0xbe, 0x43, 0xdc, 0xa4, 0x05, 0x2a, 0x7f, 0xf0, 0xc8, 0xa2, 0x01, 0x09, 0xb1, - 0x6f, 0xab, 0xad, 0x4f, 0x7e, 0x72, 0xfc, 0x23, 0xec, 0xc6, 0xc9, 0x26, 0xcb, 0x1f, 0xda, 0x5f, - 0xa7, 0x00, 0x95, 0x2d, 0x4c, 0x3a, 0x20, 0x21, 0x89, 0x78, 0x54, 0x66, 0x43, 0x7d, 0x2e, 0x03, - 0x17, 0xe1, 0xfe, 0x13, 0xe8, 0x5a, 0xd1, 0x91, 0x29, 0x5c, 0x22, 0x64, 0xf6, 0x36, 0x3f, 0x3c, - 0xb1, 0x4b, 0x37, 0xb6, 0x76, 0x9e, 0x08, 0xa8, 0xd1, 0xb1, 0xa2, 0x23, 0xf1, 0x85, 0x7e, 0x06, - 0xf0, 0x8b, 0x88, 0xfa, 0x8a, 0xb3, 0xdc, 0xf8, 0x8f, 0x4e, 0xce, 0xf9, 0x47, 0x3b, 0x8f, 0xb7, - 0x25, 0xeb, 0x2e, 0x67, 0x27, 0x79, 0x5b, 0x30, 0x1b, 0xe0, 0xf0, 0x79, 0x4c, 0x98, 0x62, 0x2f, - 0x63, 0xe1, 0xbb, 0x27, 0x67, 0xff, 0x63, 0xc9, 0x46, 0x4a, 0xe8, 0x07, 0x99, 0x5f, 0xda, 0x9f, - 0x4f, 0x41, 0x27, 0xb1, 0x8b, 0x57, 0xa0, 0x22, 0xc2, 0xe5, 0x3b, 0xcc, 0x74, 0xfc, 0x7d, 0xaa, - 0x3c, 0x7a, 0x9a, 0xc3, 0xe5, 0x53, 0xec, 0xa1, 0xbf, 0x4f, 0xb9, 0xef, 0x43, 0x62, 0xd1, 0xd0, - 0xe6, 0xf7, 0xbd, 0xe3, 0x39, 0x3c, 0xec, 0xe5, 0x5e, 0xce, 0x49, 0xf8, 0xbd, 0x04, 0x8c, 0xae, - 0xc0, 0x9c, 0xd8, 0xf6, 0x0c, 0x66, 0x3b, 0xe1, 0x49, 0xdc, 0x0c, 0xe2, 0x55, 0x98, 0x7f, 0x1e, - 0x53, 0x46, 0x4c, 0xeb, 0x00, 0x87, 0xd8, 0x62, 0x34, 0x7d, 0x11, 0xcd, 0x09, 0xf8, 0x56, 0x0a, - 0x46, 0xff, 0x07, 0x4b, 0x12, 0x95, 0x44, 0x16, 0x0e, 0x52, 0x0a, 0x12, 0xaa, 0x82, 0x79, 0x51, - 0xac, 0xde, 0x17, 0x8b, 0x5b, 0xc9, 0x1a, 0xd2, 0xa0, 0x63, 0x51, 0xcf, 0x23, 0x3e, 0x8b, 0xc4, - 0xa9, 0xda, 0x35, 0xd2, 0xdf, 0xe8, 0x0e, 0xac, 0x60, 0xd7, 0xa5, 0x2f, 0x4c, 0x41, 0x69, 0x9b, - 0x25, 0xeb, 0x66, 0x44, 0x3d, 0xa3, 0x09, 0xa4, 0xcf, 0x04, 0x8e, 0x91, 0x37, 0x54, 0xbb, 0x08, - 0xdd, 0x74, 0x1f, 0xf9, 0xe9, 0x9d, 0x09, 0x48, 0xf1, 0xad, 0x9d, 0x86, 0x7e, 0x76, 0x27, 0xb4, - 0x7f, 0xb6, 0x61, 0xa1, 0x22, 0xa9, 0xd0, 0x53, 0x00, 0x1e, 0xad, 0x32, 0xb5, 0x54, 0xb8, 0x7e, - 0xe7, 0xe4, 0xc9, 0xc9, 0xe3, 0x55, 0x82, 0x0d, 0x1e, 0xfd, 0xf2, 0x13, 0xfd, 0x1c, 0x7a, 0x22, - 0x62, 0x15, 0x77, 0x19, 0xb2, 0x1f, 0x7f, 0x03, 0xee, 0xdc, 0x56, 0xc5, 0x5e, 0xe4, 0x80, 0xfc, - 0xd6, 0xfe, 0xde, 0x82, 0x6e, 0x2a, 0x98, 0xdf, 0x33, 0x72, 0xa3, 0xc4, 0x5e, 0x47, 0xc9, 0x55, - 0x24, 0x60, 0x0f, 0x04, 0xe8, 0x7f, 0x32, 0x94, 0xb4, 0x0f, 0x00, 0x46, 0xf6, 0x57, 0x9a, 0xd0, - 0xaa, 0x34, 0x41, 0xbf, 0x0a, 0xb3, 0xdc, 0xb3, 0x0e, 0xb1, 0x77, 0x58, 0xe8, 0x04, 0xe2, 0x9a, - 0x95, 0x38, 0x91, 0x2a, 0xa6, 0x93, 0x9f, 0x9b, 0x7f, 0x18, 0x40, 0x3f, 0xdb, 0x04, 0x40, 0x5f, - 0x42, 0x2f, 0x33, 0xb4, 0x42, 0x6f, 0x95, 0x37, 0xad, 0x3c, 0x04, 0xd3, 0xde, 0x1e, 0x83, 0xa5, - 0xae, 0xc4, 0x09, 0xe4, 0xc3, 0x99, 0xd2, 0xe4, 0x07, 0x5d, 0x2b, 0x53, 0xd7, 0xcd, 0x95, 0xb4, - 0xeb, 0x8d, 0x70, 0x53, 0x79, 0x0c, 0x16, 0x2a, 0x46, 0x39, 0xe8, 0xdd, 0x31, 0x5c, 0x72, 0xe3, - 0x24, 0xed, 0xbd, 0x86, 0xd8, 0xa9, 0xd4, 0xe7, 0x80, 0xca, 0x73, 0x1e, 0x74, 0x7d, 0x2c, 0x9b, - 0xd1, 0x1c, 0x49, 0x7b, 0xb7, 0x19, 0x72, 0xad, 0xa1, 0x72, 0x02, 0x34, 0xd6, 0xd0, 0xdc, 0x8c, - 0x69, 0xac, 0xa1, 0x85, 0xb1, 0xd2, 0x04, 0x3a, 0x84, 0xf9, 0xe2, 0x74, 0x08, 0x5d, 0xad, 0x9b, - 0x66, 0x96, 0x86, 0x4f, 0xda, 0xb5, 0x26, 0xa8, 0xa9, 0x30, 0x02, 0xa7, 0xf3, 0xd3, 0x18, 0x74, - 0xa5, 0x4c, 0x5f, 0x39, 0x8f, 0xd2, 0xd6, 0xc7, 0x23, 0x66, 0x6d, 0x2a, 0x4e, 0x68, 0xaa, 0x6c, - 0xaa, 0x19, 0xff, 0x54, 0xd9, 0x54, 0x37, 0xf0, 0xd1, 0x27, 0xd0, 0x2f, 0x93, 0xb6, 0x7f, 0x61, - 0x72, 0x81, 0x36, 0xea, 0xd8, 0x54, 0x8f, 0x4e, 0xb4, 0x1b, 0x8d, 0xf1, 0x13, 0xd9, 0x37, 0x5b, - 0x3c, 0xd7, 0x33, 0x03, 0x8c, 0xaa, 0x5c, 0x2f, 0x8f, 0x44, 0xaa, 0x72, 0xbd, 0x6a, 0x0a, 0x32, - 0x81, 0xf6, 0x60, 0x36, 0x37, 0xd2, 0x40, 0xef, 0xd4, 0x51, 0xe6, 0x7b, 0x21, 0xda, 0x95, 0xb1, - 0x78, 0xa9, 0x0c, 0x33, 0x39, 0xbd, 0xd4, 0x71, 0x55, 0xab, 0x5c, 0xfe, 0xbc, 0x7a, 0x67, 0x1c, - 0x5a, 0x2e, 0x95, 0x4b, 0x83, 0x8f, 0xca, 0x54, 0xae, 0x1b, 0xac, 0x54, 0xa6, 0x72, 0xfd, 0x2c, - 0x65, 0x02, 0xfd, 0x14, 0x60, 0x34, 0x9c, 0x40, 0x97, 0xeb, 0xa8, 0xb3, 0xbb, 0xff, 0xd6, 0xf1, - 0x48, 0x29, 0xeb, 0x17, 0xb0, 0x58, 0xd5, 0x33, 0x40, 0x15, 0x89, 0x7f, 0x4c, 0x63, 0x42, 0xdb, - 0x68, 0x8a, 0x9e, 0x0a, 0xfe, 0x1c, 0x3a, 0xc9, 0x60, 0x01, 0x5d, 0x2a, 0x53, 0x17, 0x46, 0x29, - 0x9a, 0x7e, 0x1c, 0x4a, 0x26, 0x80, 0xbd, 0x24, 0x57, 0x47, 0x1d, 0xff, 0xfa, 0x5c, 0x2d, 0xcd, - 0x26, 0xea, 0x73, 0xb5, 0x3c, 0x40, 0x10, 0xe2, 0xd2, 0x60, 0xc8, 0x36, 0xc8, 0xeb, 0x83, 0xa1, - 0xa2, 0xff, 0x5f, 0x1f, 0x0c, 0x95, 0x3d, 0xf7, 0x09, 0xf4, 0x2b, 0x58, 0xaa, 0xee, 0x8b, 0xa3, - 0xda, 0x8c, 0xaf, 0xe9, 0xcf, 0x6b, 0x37, 0x9b, 0x13, 0xa4, 0xe2, 0x5f, 0x25, 0xe7, 0x53, 0xa1, - 0x2f, 0x5e, 0x7f, 0x3e, 0x55, 0x77, 0xe7, 0xb5, 0x1b, 0x8d, 0xf1, 0xcb, 0xa9, 0x97, 0x6d, 0x40, - 0xd7, 0x7b, 0xbb, 0xa2, 0xd7, 0x5e, 0xef, 0xed, 0xca, 0x9e, 0xb6, 0xc8, 0x8f, 0xaa, 0xe6, 0x72, - 0x55, 0x7e, 0x1c, 0xd3, 0xfd, 0xd6, 0x36, 0x9a, 0xa2, 0xe7, 0xae, 0xef, 0x72, 0xf7, 0x18, 0x8d, - 0xd5, 0x3f, 0x77, 0x32, 0xbf, 0xd7, 0x10, 0xbb, 0x7e, 0x77, 0x93, 0x93, 0x7a, 0xac, 0x01, 0x85, - 0x13, 0xfb, 0x46, 0x63, 0xfc, 0x54, 0x76, 0x90, 0x8c, 0x8c, 0x33, 0x9d, 0x5f, 0x74, 0x6d, 0x0c, - 0x9f, 0x4c, 0xe7, 0x5a, 0xbb, 0xde, 0x08, 0xb7, 0x2a, 0x7b, 0xb3, 0xbd, 0xd8, 0xe3, 0xe2, 0xa9, - 0xd4, 0x40, 0x3e, 0x2e, 0x9e, 0x2a, 0xda, 0xbb, 0x13, 0xe8, 0x37, 0xa3, 0x59, 0x60, 0xb9, 0x53, - 0x84, 0x36, 0x6b, 0xcf, 0x82, 0xda, 0x8e, 0x97, 0x76, 0xeb, 0x44, 0x34, 0xa9, 0x22, 0x9f, 0xc2, - 0x94, 0x78, 0x6b, 0xa1, 0xd5, 0xe3, 0x1f, 0x61, 0xda, 0xc5, 0xea, 0xf5, 0xf4, 0x29, 0xc1, 0x3d, - 0xb9, 0x37, 0x2d, 0xfe, 0x22, 0x77, 0xeb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x81, 0x26, 0xf7, - 0x49, 0x39, 0x27, 0x00, 0x00, + 0xf5, 0x5a, 0xaf, 0x64, 0xed, 0xbe, 0x5d, 0x59, 0x72, 0x4b, 0x91, 0x37, 0x23, 0x4b, 0x56, 0x26, + 0x5f, 0x92, 0x13, 0xcb, 0x8e, 0xf2, 0xfb, 0x91, 0x90, 0x10, 0xc0, 0x96, 0x6c, 0x30, 0x89, 0xe5, + 0x64, 0xe4, 0x98, 0x40, 0x52, 0x4c, 0xb5, 0x66, 0x5a, 0xd6, 0xa0, 0x99, 0xe9, 0xf1, 0x4c, 0xaf, + 0xe2, 0x75, 0xc1, 0x29, 0x1c, 0xb8, 0xf0, 0x07, 0x70, 0xe6, 0xce, 0x89, 0x2a, 0xaa, 0xb8, 0x72, + 0xc9, 0x1f, 0x00, 0x57, 0x2e, 0x9c, 0x39, 0x70, 0xa3, 0x8a, 0x0b, 0xd5, 0x5f, 0xb3, 0xf3, 0xa9, + 0x1d, 0xc5, 0xae, 0xa2, 0xb8, 0xf5, 0xbc, 0x7e, 0x1f, 0xfd, 0x5e, 0xbf, 0xf7, 0xfa, 0x75, 0xbf, + 0x81, 0xc5, 0x13, 0xea, 0x0f, 0x03, 0x62, 0x27, 0x24, 0x3e, 0x21, 0xf1, 0x56, 0x14, 0x53, 0x46, + 0xd1, 0x42, 0x0e, 0x68, 0x47, 0x07, 0xe6, 0x75, 0x40, 0xb7, 0x30, 0x73, 0x8e, 0x76, 0x89, 0x4f, + 0x18, 0xb1, 0xc8, 0xe3, 0x21, 0x49, 0x18, 0x7a, 0x11, 0x3a, 0x87, 0x9e, 0x4f, 0x6c, 0xcf, 0x4d, + 0x06, 0xad, 0xf5, 0xf6, 0x46, 0xd7, 0x9a, 0xe5, 0xdf, 0x77, 0xdd, 0xc4, 0xbc, 0x0f, 0x8b, 0x39, + 0x82, 0x24, 0xa2, 0x61, 0x42, 0xd0, 0xbb, 0x30, 0x1b, 0x93, 0x64, 0xe8, 0x33, 0x49, 0xd0, 0xdb, + 0x5e, 0xdb, 0x2a, 0xca, 0xda, 0x4a, 0x49, 0x86, 0x3e, 0xb3, 0x34, 0xba, 0xf9, 0x55, 0x0b, 0xfa, + 0xd9, 0x19, 0x74, 0x09, 0x66, 0x95, 0xf0, 0x41, 0x6b, 0xbd, 0xb5, 0xd1, 0xb5, 0xce, 0x4b, 0xd9, + 0x68, 0x19, 0xce, 0x27, 0x0c, 0xb3, 0x61, 0x32, 0x38, 0xb7, 0xde, 0xda, 0x98, 0xb1, 0xd4, 0x17, + 0x5a, 0x82, 0x19, 0x12, 0xc7, 0x34, 0x1e, 0xb4, 0x05, 0xba, 0xfc, 0x40, 0x08, 0xa6, 0x13, 0xef, + 0x29, 0x19, 0x4c, 0xaf, 0xb7, 0x36, 0xe6, 0x2c, 0x31, 0x46, 0x03, 0x98, 0x3d, 0x21, 0x71, 0xe2, + 0xd1, 0x70, 0x30, 0x23, 0xc0, 0xfa, 0xd3, 0x9c, 0x85, 0x99, 0xdb, 0x41, 0xc4, 0x46, 0xe6, 0x3b, + 0x30, 0x78, 0x88, 0x9d, 0xe1, 0x30, 0x78, 0x28, 0x96, 0xbf, 0x73, 0x44, 0x9c, 0x63, 0x6d, 0x96, + 0x15, 0xe8, 0x2a, 0xa5, 0xd4, 0xda, 0xe6, 0xac, 0x8e, 0x04, 0xdc, 0x75, 0xcd, 0xef, 0xc3, 0x8b, + 0x15, 0x84, 0xca, 0x3c, 0x2f, 0xc3, 0xdc, 0x23, 0x1c, 0x1f, 0xe0, 0x47, 0xc4, 0x8e, 0x31, 0xf3, + 0xa8, 0xa0, 0x6e, 0x59, 0x7d, 0x05, 0xb4, 0x38, 0xcc, 0xfc, 0x1c, 0x8c, 0x1c, 0x07, 0x1a, 0x44, + 0xd8, 0x61, 0x4d, 0x84, 0xa3, 0x75, 0xe8, 0x45, 0x31, 0xc1, 0xbe, 0x4f, 0x1d, 0xcc, 0x88, 0xb0, + 0x4f, 0xdb, 0xca, 0x82, 0xcc, 0x55, 0x58, 0xa9, 0x64, 0x2e, 0x17, 0x68, 0xbe, 0x5b, 0x58, 0x3d, + 0x0d, 0x02, 0xaf, 0x91, 0x68, 0xf3, 0x72, 0x69, 0xd5, 0x82, 0x52, 0xf1, 0xfd, 0x76, 0x61, 0xd6, + 0x27, 0x38, 0x1c, 0x46, 0x8d, 0x18, 0x17, 0x57, 0xac, 0x49, 0x53, 0xce, 0x97, 0xa4, 0xdb, 0xec, + 0x50, 0xdf, 0x27, 0x0e, 0xf3, 0x68, 0xa8, 0xd9, 0xae, 0x01, 0x38, 0x29, 0x50, 0x39, 0x51, 0x06, + 0x62, 0x1a, 0x30, 0x28, 0x93, 0x2a, 0xb6, 0x7f, 0x6b, 0xc1, 0x0b, 0x37, 0x95, 0xd1, 0xa4, 0xe0, + 0x46, 0x1b, 0x90, 0x17, 0x79, 0xae, 0x28, 0xb2, 0xb8, 0x41, 0xed, 0xd2, 0x06, 0x71, 0x8c, 0x98, + 0x44, 0xbe, 0xe7, 0x60, 0xc1, 0x62, 0x5a, 0xb0, 0xc8, 0x82, 0xd0, 0x02, 0xb4, 0x19, 0xf3, 0x85, + 0xe7, 0x76, 0x2d, 0x3e, 0x44, 0xdb, 0xb0, 0x1c, 0x90, 0x80, 0xc6, 0x23, 0x3b, 0xc0, 0x91, 0x1d, + 0xe0, 0x27, 0x36, 0x77, 0x73, 0x3b, 0x38, 0x18, 0x9c, 0x17, 0xeb, 0x43, 0x72, 0xf6, 0x1e, 0x8e, + 0xee, 0xe1, 0x27, 0xfb, 0xde, 0x53, 0x72, 0xef, 0xc0, 0x1c, 0xc0, 0x72, 0x51, 0x3f, 0xa5, 0xfa, + 0xb7, 0xe0, 0x92, 0x84, 0xec, 0x8f, 0x42, 0x67, 0x5f, 0xc4, 0x56, 0xa3, 0x8d, 0xfa, 0x77, 0x0b, + 0x06, 0x65, 0x42, 0xe5, 0xf9, 0xcf, 0x6a, 0xb5, 0x33, 0xdb, 0xe4, 0x0a, 0xf4, 0x18, 0xf6, 0x7c, + 0x9b, 0x1e, 0x1e, 0x26, 0x84, 0x09, 0x43, 0x4c, 0x5b, 0xc0, 0x41, 0xf7, 0x05, 0x04, 0x6d, 0xc2, + 0x82, 0x23, 0xbd, 0xdf, 0x8e, 0xc9, 0x89, 0x27, 0xb2, 0xc1, 0xac, 0x58, 0xd8, 0xbc, 0xa3, 0xa3, + 0x42, 0x82, 0x91, 0x09, 0x73, 0x9e, 0xfb, 0xc4, 0x16, 0xe9, 0x48, 0x24, 0x93, 0x8e, 0xe0, 0xd6, + 0xf3, 0xdc, 0x27, 0x77, 0x3c, 0x9f, 0x70, 0x8b, 0x9a, 0x0f, 0xe1, 0xb2, 0x54, 0xfe, 0x6e, 0xe8, + 0xc4, 0x24, 0x20, 0x21, 0xc3, 0xfe, 0x0e, 0x8d, 0x46, 0x8d, 0xdc, 0xe6, 0x45, 0xe8, 0x24, 0x5e, + 0xe8, 0x10, 0x3b, 0x94, 0x49, 0x6d, 0xda, 0x9a, 0x15, 0xdf, 0x7b, 0x89, 0x79, 0x0b, 0x56, 0x6b, + 0xf8, 0x2a, 0xcb, 0xbe, 0x04, 0x7d, 0xb1, 0x30, 0x87, 0x86, 0x8c, 0x84, 0x4c, 0xf0, 0xee, 0x5b, + 0x3d, 0x0e, 0xdb, 0x91, 0x20, 0xf3, 0x2d, 0x40, 0x92, 0xc7, 0x3d, 0x3a, 0x0c, 0x9b, 0x85, 0xf3, + 0x0b, 0xb0, 0x98, 0x23, 0x51, 0xbe, 0xf1, 0x36, 0x2c, 0x49, 0xf0, 0xa7, 0x61, 0xd0, 0x98, 0xd7, + 0x25, 0x78, 0xa1, 0x40, 0xa4, 0xb8, 0x6d, 0x6b, 0x21, 0xf9, 0x63, 0xe7, 0x54, 0x66, 0xcb, 0x7a, + 0x05, 0xf9, 0x93, 0x47, 0x64, 0x2e, 0xb9, 0x60, 0x1c, 0x1f, 0x5b, 0x04, 0xbb, 0x34, 0xf4, 0x47, + 0x8d, 0x33, 0x57, 0x05, 0xa5, 0xe2, 0xfb, 0xfb, 0x16, 0x5c, 0xd4, 0x29, 0xad, 0xe1, 0x6e, 0x9e, + 0xd1, 0x9d, 0xdb, 0xb5, 0xee, 0x3c, 0x3d, 0x76, 0xe7, 0x0d, 0x58, 0x48, 0xe8, 0x30, 0x76, 0x88, + 0xed, 0x62, 0x86, 0xed, 0x90, 0xba, 0x44, 0x79, 0xfb, 0x05, 0x09, 0xdf, 0xc5, 0x0c, 0xef, 0x51, + 0x97, 0x98, 0xdf, 0xd3, 0x9b, 0x9d, 0xf3, 0x92, 0x4d, 0xb8, 0xe8, 0xe3, 0x84, 0xd9, 0x38, 0x8a, + 0x48, 0xe8, 0xda, 0x98, 0x71, 0x57, 0x6b, 0x09, 0x57, 0xbb, 0xc0, 0x27, 0x6e, 0x0a, 0xf8, 0x4d, + 0xb6, 0x97, 0x98, 0x7f, 0x69, 0xc1, 0x3c, 0xa7, 0xe5, 0xae, 0xdd, 0x48, 0xdf, 0x05, 0x68, 0x93, + 0x27, 0x4c, 0x29, 0xca, 0x87, 0xe8, 0x3a, 0x2c, 0xaa, 0x18, 0xf2, 0x68, 0x38, 0x0e, 0xaf, 0xb6, + 0xcc, 0x46, 0xe3, 0xa9, 0x34, 0xc2, 0xae, 0x40, 0x2f, 0x61, 0x34, 0xd2, 0xd1, 0x3a, 0x2d, 0xa3, + 0x95, 0x83, 0x54, 0xb4, 0xe6, 0x6d, 0x3a, 0x53, 0x61, 0xd3, 0xbe, 0x97, 0xd8, 0xc4, 0xb1, 0xe5, + 0xaa, 0x44, 0xbc, 0x77, 0x2c, 0xf0, 0x92, 0xdb, 0x8e, 0xb4, 0x86, 0xf9, 0xff, 0xb0, 0x30, 0xd6, + 0xaa, 0x79, 0xec, 0x7c, 0xd5, 0xd2, 0xe9, 0xf0, 0x01, 0xf6, 0xfc, 0x7d, 0x12, 0xba, 0x24, 0x7e, + 0xc6, 0x98, 0x46, 0x37, 0x60, 0xc9, 0x73, 0x7d, 0x62, 0x33, 0x2f, 0x20, 0x74, 0xc8, 0xec, 0x84, + 0x38, 0x34, 0x74, 0x13, 0x6d, 0x1f, 0x3e, 0xf7, 0x40, 0x4e, 0xed, 0xcb, 0x19, 0xf3, 0x57, 0x69, + 0x6e, 0xcd, 0xae, 0x62, 0x5c, 0x55, 0x84, 0x84, 0x70, 0x86, 0x47, 0x04, 0xbb, 0x24, 0x56, 0x6a, + 0xf4, 0x25, 0xf0, 0x87, 0x02, 0xc6, 0x2d, 0xac, 0x90, 0x0e, 0xa8, 0x3b, 0x12, 0x2b, 0xea, 0x5b, + 0x20, 0x41, 0xb7, 0xa8, 0x3b, 0x12, 0x49, 0x2e, 0xb1, 0x85, 0x93, 0x38, 0x47, 0xc3, 0xf0, 0x58, + 0xac, 0xa6, 0x63, 0xf5, 0xbc, 0xe4, 0x23, 0x9c, 0xb0, 0x1d, 0x0e, 0x32, 0xff, 0xd8, 0xd2, 0x51, + 0xc6, 0x97, 0x61, 0x11, 0x87, 0x78, 0x27, 0xff, 0x05, 0x73, 0x70, 0x0a, 0x15, 0x0d, 0xb9, 0xea, + 0x52, 0x05, 0x0c, 0x92, 0x73, 0xea, 0x2c, 0x12, 0x33, 0xe3, 0x20, 0xcf, 0x2f, 0x5c, 0x05, 0xf9, + 0x17, 0x3a, 0xc9, 0xde, 0x76, 0xf6, 0x8f, 0x70, 0xec, 0x26, 0x3f, 0x20, 0x21, 0x89, 0x31, 0x7b, + 0x2e, 0x87, 0xbe, 0xb9, 0x0e, 0x6b, 0x75, 0xdc, 0x95, 0xfc, 0xcf, 0xf5, 0xe1, 0xa1, 0x31, 0x2c, + 0x72, 0x30, 0xf4, 0x7c, 0xf7, 0xb9, 0x88, 0xff, 0xb0, 0xa8, 0x5c, 0xca, 0x5c, 0xf9, 0xcf, 0x55, + 0xb8, 0x18, 0x0b, 0x10, 0xb3, 0x13, 0x8e, 0x90, 0xd6, 0xfb, 0x73, 0xd6, 0xbc, 0x9a, 0x10, 0x84, + 0xbc, 0xee, 0xff, 0x73, 0xea, 0x01, 0x9a, 0xdb, 0x73, 0x4b, 0x8b, 0x2b, 0xd0, 0x1d, 0x8b, 0x6f, + 0x0b, 0xf1, 0x9d, 0x44, 0xc9, 0xe5, 0xde, 0xe9, 0xd0, 0x68, 0x64, 0x13, 0x47, 0x9e, 0xc3, 0x62, + 0xab, 0x3b, 0x56, 0x8f, 0x03, 0x6f, 0x3b, 0xe2, 0x18, 0x3e, 0x43, 0x8e, 0x4c, 0xbd, 0x21, 0xaf, + 0x84, 0xda, 0x8d, 0x2f, 0x61, 0x25, 0x3f, 0xdb, 0xfc, 0x78, 0x7a, 0x26, 0x25, 0xcd, 0xb5, 0xa2, + 0x1b, 0x14, 0xce, 0xb8, 0x93, 0xe2, 0xb2, 0x1b, 0x9f, 0xe7, 0xcf, 0xb6, 0xae, 0xd5, 0xa2, 0x41, + 0xf2, 0x45, 0xc1, 0x67, 0xc5, 0x65, 0x9f, 0xa1, 0x38, 0x38, 0x5d, 0xf0, 0x95, 0xa2, 0xeb, 0x16, + 0x2b, 0x88, 0xdf, 0xa6, 0x79, 0x51, 0x61, 0xf0, 0xf3, 0xbb, 0x71, 0x3e, 0x52, 0x72, 0x85, 0x39, + 0xe6, 0xac, 0x59, 0x25, 0x96, 0x5f, 0x30, 0xd5, 0x39, 0x24, 0xeb, 0x73, 0xf5, 0x95, 0xbb, 0x4a, + 0xb6, 0xd5, 0x55, 0x52, 0x5f, 0x91, 0x8f, 0xc9, 0x48, 0xf8, 0xda, 0xb4, 0xbc, 0x22, 0x7f, 0x48, + 0x46, 0xe6, 0x5e, 0x21, 0x52, 0xe4, 0xd2, 0x54, 0xcc, 0x21, 0x98, 0xe6, 0x4e, 0xaa, 0x52, 0xb5, + 0x18, 0xa3, 0x55, 0x00, 0x2f, 0xb1, 0x5d, 0xb1, 0xe7, 0x72, 0x51, 0x1d, 0xab, 0xeb, 0x29, 0x27, + 0x70, 0xcd, 0xdf, 0x64, 0x42, 0xef, 0x96, 0x4f, 0x0f, 0x9e, 0xa3, 0x57, 0x66, 0xb5, 0x68, 0xe7, + 0xb4, 0xc8, 0xde, 0x95, 0xa7, 0xf3, 0x77, 0xe5, 0x4c, 0x10, 0x65, 0x97, 0xa3, 0x76, 0xe6, 0x3d, + 0x58, 0xe1, 0x0a, 0x4b, 0x0c, 0x51, 0x25, 0x37, 0xbf, 0x49, 0xfc, 0xe3, 0x1c, 0x5c, 0xae, 0x26, + 0x6e, 0x72, 0x9b, 0x78, 0x1f, 0x8c, 0xb4, 0x5a, 0xe7, 0x47, 0x4a, 0xc2, 0x70, 0x10, 0xa5, 0x87, + 0x8a, 0x3c, 0x7b, 0x2e, 0xa9, 0xd2, 0xfd, 0x81, 0x9e, 0xd7, 0x27, 0x4b, 0xa9, 0xd4, 0x6f, 0x97, + 0x4a, 0x7d, 0x2e, 0xc0, 0xc5, 0xac, 0x4e, 0x80, 0xac, 0x5d, 0x2e, 0xb9, 0x98, 0xd5, 0x09, 0x48, + 0x89, 0x85, 0x00, 0xe9, 0x35, 0x3d, 0x85, 0x2f, 0x04, 0xac, 0x02, 0xa8, 0xb2, 0x64, 0x18, 0xea, + 0xab, 0x4b, 0x57, 0x16, 0x25, 0xc3, 0xb0, 0xb6, 0xba, 0x9a, 0xad, 0xad, 0xae, 0xf2, 0xdb, 0xdf, + 0x29, 0x9d, 0x10, 0x9f, 0x01, 0xec, 0x7a, 0xc9, 0xb1, 0x34, 0x32, 0x2f, 0xe7, 0x5c, 0x2f, 0x56, + 0xf7, 0x65, 0x3e, 0xe4, 0x10, 0xec, 0xfb, 0xca, 0x74, 0x7c, 0xc8, 0xdd, 0x77, 0x98, 0x10, 0x57, + 0x59, 0x47, 0x8c, 0x39, 0xec, 0x30, 0x26, 0x44, 0x19, 0x40, 0x8c, 0xcd, 0xdf, 0xb5, 0xa0, 0x7b, + 0x8f, 0x04, 0x8a, 0xf3, 0x1a, 0xc0, 0x23, 0x1a, 0xd3, 0x21, 0xf3, 0x42, 0x22, 0xab, 0xcf, 0x19, + 0x2b, 0x03, 0xf9, 0xe6, 0x72, 0x44, 0x68, 0x12, 0xff, 0x50, 0x19, 0x53, 0x8c, 0x39, 0xec, 0x88, + 0xe0, 0x48, 0xd9, 0x4f, 0x8c, 0xd1, 0x12, 0xcc, 0x24, 0x0c, 0x3b, 0xc7, 0xc2, 0x58, 0xd3, 0x96, + 0xfc, 0x30, 0xff, 0xd4, 0x02, 0xb0, 0x48, 0x40, 0x99, 0xf0, 0x35, 0x5e, 0x15, 0x1e, 0x60, 0xe7, + 0x98, 0xd7, 0xc9, 0x6c, 0x14, 0x11, 0x65, 0x89, 0x9e, 0x82, 0x3d, 0x18, 0x45, 0x62, 0x87, 0x34, + 0x8a, 0xca, 0x1f, 0x5d, 0xab, 0xab, 0x20, 0xb2, 0x22, 0xd6, 0xa1, 0xd4, 0xb5, 0xf8, 0x30, 0x93, + 0x53, 0xe4, 0xb2, 0x75, 0x4e, 0x59, 0x81, 0x6e, 0xd1, 0x15, 0x44, 0x28, 0x0a, 0x3f, 0x78, 0x19, + 0xe6, 0x02, 0xea, 0x7a, 0x87, 0x1e, 0x71, 0x85, 0xa3, 0x29, 0x55, 0xfa, 0x1a, 0xc8, 0x9d, 0xcb, + 0xdc, 0x85, 0x0b, 0xaa, 0xb2, 0xf1, 0x48, 0x7c, 0x37, 0x3c, 0xa4, 0x68, 0x1b, 0x66, 0x38, 0x0b, + 0xfd, 0x04, 0x77, 0xb9, 0xfc, 0x04, 0x37, 0x56, 0xd6, 0x92, 0xa8, 0xe6, 0xd7, 0x2d, 0x58, 0x1f, + 0xb3, 0xe1, 0xc7, 0xe1, 0x2e, 0x66, 0x0f, 0xa8, 0x44, 0x7c, 0x2e, 0x39, 0xe6, 0x5d, 0x18, 0xb8, + 0x24, 0x61, 0x5e, 0x28, 0xae, 0x38, 0xb6, 0x36, 0x5f, 0x88, 0x03, 0xa2, 0x0c, 0xb5, 0x9c, 0x99, + 0xbf, 0x25, 0xa7, 0xf7, 0x70, 0x40, 0xd0, 0x35, 0x58, 0x3c, 0x26, 0x24, 0xb2, 0x7d, 0xea, 0x60, + 0xdf, 0xd6, 0xd1, 0xa3, 0x2a, 0x80, 0x05, 0x3e, 0xf5, 0x11, 0x9f, 0xd9, 0x95, 0x11, 0x64, 0x26, + 0xf0, 0xd2, 0x29, 0x9a, 0xa8, 0x0c, 0x72, 0x19, 0xba, 0x51, 0x4c, 0x1d, 0x92, 0x70, 0xef, 0x6a, + 0x89, 0x84, 0x3e, 0x06, 0xa0, 0x1b, 0xb0, 0x98, 0x7e, 0x7c, 0x4c, 0x62, 0x87, 0xdf, 0xba, 0x1f, + 0xc9, 0xf7, 0xb4, 0x73, 0x56, 0xd5, 0x94, 0xf9, 0xaf, 0x3e, 0xf4, 0x3f, 0x19, 0x92, 0x78, 0x94, + 0x79, 0x7c, 0x4a, 0x88, 0xd2, 0x5d, 0xbf, 0x9e, 0x66, 0x20, 0x3c, 0x0f, 0x1c, 0xc6, 0x34, 0xb0, + 0xd3, 0x07, 0xd6, 0x73, 0x02, 0xa5, 0xc7, 0x81, 0x77, 0xe4, 0x23, 0x2b, 0xfa, 0x00, 0xce, 0x1f, + 0x7a, 0x3e, 0x23, 0xf2, 0x49, 0xb3, 0xb7, 0xfd, 0x6a, 0x79, 0x27, 0xb3, 0x32, 0xb7, 0xee, 0x08, + 0x64, 0x4b, 0x11, 0xa1, 0x03, 0x58, 0xf4, 0xc2, 0x48, 0x14, 0xd4, 0xb1, 0x87, 0x7d, 0xef, 0xe9, + 0xf8, 0xf9, 0xa4, 0xb7, 0xfd, 0xd6, 0x04, 0x5e, 0x77, 0x39, 0xe5, 0x7e, 0x96, 0xd0, 0x42, 0x5e, + 0x09, 0x86, 0x08, 0x2c, 0xd1, 0x21, 0x2b, 0x0b, 0x99, 0x11, 0x42, 0xb6, 0x27, 0x08, 0xb9, 0x2f, + 0x48, 0xf3, 0x52, 0x16, 0x69, 0x19, 0x68, 0xec, 0xc1, 0x79, 0xa9, 0x1c, 0x8f, 0xe0, 0x43, 0x8f, + 0xf8, 0xfa, 0x51, 0x58, 0x7e, 0xf0, 0x53, 0x8a, 0x46, 0x24, 0xc6, 0xa1, 0x0e, 0x46, 0xfd, 0xc9, + 0xf1, 0x4f, 0xb0, 0x3f, 0xd4, 0x3e, 0x26, 0x3f, 0x8c, 0xbf, 0xce, 0x00, 0x2a, 0x6b, 0xa8, 0xdf, + 0x84, 0x62, 0x92, 0xf0, 0xbc, 0x99, 0x8d, 0xfe, 0xf9, 0x0c, 0x5c, 0x64, 0x80, 0x1f, 0x43, 0xd7, + 0x49, 0x4e, 0x6c, 0x61, 0x12, 0x21, 0xb3, 0xb7, 0xfd, 0xde, 0x99, 0x4d, 0xba, 0xb5, 0xb3, 0xff, + 0x50, 0x40, 0xad, 0x8e, 0x93, 0x9c, 0x88, 0x11, 0xfa, 0x29, 0xc0, 0xcf, 0x13, 0x1a, 0x2a, 0xce, + 0x72, 0xe3, 0xdf, 0x3f, 0x3b, 0xe7, 0x1f, 0xed, 0xdf, 0xdf, 0x93, 0xac, 0xbb, 0x9c, 0x9d, 0xe4, + 0xed, 0xc0, 0x5c, 0x84, 0xe3, 0xc7, 0x43, 0xc2, 0x14, 0x7b, 0xe9, 0x0b, 0xdf, 0x3d, 0x3b, 0xfb, + 0x8f, 0x25, 0x1b, 0x29, 0xa1, 0x1f, 0x65, 0xbe, 0x8c, 0xaf, 0xcf, 0x41, 0x47, 0xeb, 0xc5, 0x6b, + 0x72, 0xe1, 0xe1, 0xf2, 0x66, 0x6a, 0x7b, 0xe1, 0x21, 0x55, 0x16, 0xbd, 0xc0, 0xe1, 0xf2, 0x72, + 0x2a, 0xb2, 0xd6, 0x26, 0x2c, 0xc4, 0xc4, 0xa1, 0xb1, 0xcb, 0x2b, 0x20, 0x2f, 0xf0, 0xb8, 0xdb, + 0xcb, 0xbd, 0x9c, 0x97, 0xf0, 0x5d, 0x0d, 0x46, 0xaf, 0xc3, 0xbc, 0xd8, 0xf6, 0x0c, 0x66, 0x5b, + 0xf3, 0x24, 0x7e, 0x06, 0x71, 0x13, 0x16, 0x1e, 0x0f, 0x29, 0x23, 0xb6, 0x73, 0x84, 0x63, 0xec, + 0x30, 0x9a, 0xde, 0x11, 0xe7, 0x05, 0x7c, 0x27, 0x05, 0xa3, 0xff, 0x83, 0x65, 0x89, 0x4a, 0x12, + 0x07, 0x47, 0x29, 0x05, 0x89, 0xd5, 0x15, 0x62, 0x49, 0xcc, 0xde, 0x16, 0x93, 0x3b, 0x7a, 0x0e, + 0x19, 0xd0, 0x71, 0x68, 0x10, 0x90, 0x90, 0x25, 0x22, 0x39, 0x77, 0xad, 0xf4, 0x1b, 0xdd, 0x84, + 0x55, 0xec, 0xfb, 0xf4, 0x4b, 0x5b, 0x50, 0xba, 0x76, 0x49, 0xbb, 0x59, 0x91, 0xc0, 0x0c, 0x81, + 0xf4, 0x89, 0xc0, 0xb1, 0xf2, 0x8a, 0x1a, 0x57, 0xa0, 0x9b, 0xee, 0x23, 0x3f, 0xcf, 0x32, 0x0e, + 0x29, 0xc6, 0xc6, 0x05, 0xe8, 0x67, 0x77, 0xc2, 0xf8, 0x67, 0x1b, 0x16, 0x2b, 0x82, 0x0a, 0x7d, + 0x0e, 0xc0, 0xbd, 0x55, 0x86, 0x96, 0x72, 0xd7, 0xef, 0x9c, 0x3d, 0x38, 0xb9, 0xbf, 0x4a, 0xb0, + 0xc5, 0xbd, 0x5f, 0x0e, 0xd1, 0xcf, 0xa0, 0x27, 0x3c, 0x56, 0x71, 0x97, 0x2e, 0xfb, 0xc1, 0x37, + 0xe0, 0xce, 0x75, 0x55, 0xec, 0x45, 0x0c, 0xc8, 0xb1, 0xf1, 0xf7, 0x16, 0x74, 0x53, 0xc1, 0xfc, + 0x74, 0x96, 0x1b, 0x25, 0xf6, 0x3a, 0xd1, 0xa7, 0xb3, 0x80, 0xdd, 0x11, 0xa0, 0xff, 0x49, 0x57, + 0x32, 0xde, 0x01, 0x18, 0xeb, 0x5f, 0xa9, 0x42, 0xab, 0x52, 0x05, 0x73, 0x13, 0xe6, 0xb8, 0x65, + 0x3d, 0xe2, 0xee, 0xb3, 0xd8, 0x8b, 0x44, 0x7b, 0x4b, 0xe2, 0x24, 0xea, 0x7a, 0xa1, 0x3f, 0xb7, + 0xff, 0x30, 0x80, 0x7e, 0xf6, 0x59, 0x04, 0x7d, 0x01, 0xbd, 0x4c, 0x1b, 0x0f, 0xbd, 0x52, 0xde, + 0xb4, 0x72, 0x5b, 0xd0, 0x78, 0x75, 0x02, 0x96, 0xba, 0x01, 0x4c, 0xa1, 0x10, 0x2e, 0x96, 0x7a, + 0x61, 0xe8, 0x6a, 0x99, 0xba, 0xae, 0xd3, 0x66, 0xbc, 0xd1, 0x08, 0x37, 0x95, 0xc7, 0x60, 0xb1, + 0xa2, 0xb9, 0x85, 0xde, 0x9c, 0xc0, 0x25, 0xd7, 0x60, 0x33, 0xae, 0x35, 0xc4, 0x4e, 0xa5, 0x3e, + 0x06, 0x54, 0xee, 0x7c, 0xa1, 0x37, 0x26, 0xb2, 0x19, 0x77, 0xd6, 0x8c, 0x37, 0x9b, 0x21, 0xd7, + 0x2a, 0x2a, 0x7b, 0x62, 0x13, 0x15, 0xcd, 0x75, 0xdd, 0x26, 0x2a, 0x5a, 0x68, 0xb4, 0x4d, 0xa1, + 0x63, 0x58, 0x28, 0xf6, 0xcb, 0xd0, 0x66, 0x5d, 0x7f, 0xb7, 0xd4, 0x8e, 0x33, 0xae, 0x36, 0x41, + 0x4d, 0x85, 0x11, 0xb8, 0x90, 0xef, 0x4f, 0xa1, 0xd7, 0xcb, 0xf4, 0x95, 0x1d, 0x3a, 0x63, 0x63, + 0x32, 0x62, 0x56, 0xa7, 0x62, 0xcf, 0xaa, 0x4a, 0xa7, 0x9a, 0x86, 0x58, 0x95, 0x4e, 0x75, 0x2d, + 0x30, 0x73, 0x0a, 0xfd, 0x42, 0x37, 0x42, 0x0a, 0xbd, 0x1c, 0xb4, 0x55, 0xc7, 0xa6, 0xba, 0x99, + 0x64, 0x5c, 0x6f, 0x8c, 0xaf, 0x65, 0xdf, 0x68, 0xf1, 0x58, 0xcf, 0xb4, 0x74, 0xaa, 0x62, 0xbd, + 0xdc, 0x24, 0xaa, 0x8a, 0xf5, 0xaa, 0xbe, 0xd0, 0x14, 0x3a, 0x80, 0xb9, 0x5c, 0x93, 0x07, 0xbd, + 0x56, 0x47, 0x99, 0x7f, 0x1d, 0x32, 0x5e, 0x9f, 0x88, 0x97, 0xca, 0xb0, 0x75, 0xf6, 0x52, 0xe9, + 0xaa, 0x76, 0x71, 0xf9, 0x7c, 0xf5, 0xda, 0x24, 0xb4, 0x5c, 0x28, 0x97, 0x5a, 0x41, 0x95, 0xa1, + 0x5c, 0xd7, 0x6a, 0xaa, 0x0c, 0xe5, 0xfa, 0xee, 0xd2, 0x14, 0xfa, 0x09, 0xc0, 0xb8, 0x5d, 0x83, + 0x5e, 0xae, 0xa3, 0xce, 0xee, 0xfe, 0x2b, 0xa7, 0x23, 0xa5, 0xac, 0xbf, 0x84, 0xa5, 0xaa, 0x57, + 0x14, 0x74, 0xad, 0xea, 0x42, 0x58, 0xfb, 0x54, 0x63, 0x6c, 0x35, 0x45, 0x4f, 0x05, 0x7f, 0x0a, + 0x1d, 0xdd, 0x6a, 0x41, 0x2f, 0x95, 0xa9, 0x0b, 0xcd, 0x25, 0xc3, 0x3c, 0x0d, 0x25, 0xe3, 0xc0, + 0x81, 0x8e, 0xd5, 0x71, 0x0f, 0xa4, 0x3e, 0x56, 0x4b, 0xdd, 0x9a, 0xfa, 0x58, 0x2d, 0xb7, 0x54, + 0x84, 0xb8, 0xd4, 0x19, 0xb2, 0x2d, 0x83, 0x7a, 0x67, 0xa8, 0xe8, 0x88, 0xd4, 0x3b, 0x43, 0x65, + 0x17, 0x62, 0x0a, 0xfd, 0x12, 0x96, 0xab, 0x3b, 0x05, 0xa8, 0x36, 0xe2, 0x6b, 0x3a, 0x16, 0xc6, + 0x8d, 0xe6, 0x04, 0xa9, 0xf8, 0xa7, 0x3a, 0x3f, 0x15, 0x3a, 0x05, 0xf5, 0xf9, 0xa9, 0xba, 0x5f, + 0x61, 0x5c, 0x6f, 0x8c, 0x5f, 0x0e, 0xbd, 0xec, 0x93, 0x7c, 0xbd, 0xb5, 0x2b, 0xba, 0x0f, 0xf5, + 0xd6, 0xae, 0x7c, 0xe5, 0x17, 0xf1, 0x51, 0xf5, 0xdc, 0x5e, 0x15, 0x1f, 0xa7, 0xf4, 0x03, 0x8c, + 0xad, 0xa6, 0xe8, 0xb9, 0xe3, 0xbb, 0xfc, 0x9e, 0x8e, 0x26, 0xae, 0x3f, 0x97, 0x99, 0xaf, 0x35, + 0xc4, 0xae, 0xdf, 0x5d, 0x9d, 0xa9, 0x27, 0x2a, 0x50, 0xc8, 0xd8, 0xd7, 0x1b, 0xe3, 0xa7, 0xb2, + 0x23, 0xdd, 0x44, 0xcf, 0xbc, 0x85, 0xa3, 0xab, 0x13, 0xf8, 0x64, 0xde, 0xf2, 0x8d, 0x37, 0x1a, + 0xe1, 0x56, 0x45, 0x6f, 0xf6, 0x75, 0xfa, 0x34, 0x7f, 0x2a, 0x3d, 0xa9, 0x9f, 0xe6, 0x4f, 0x15, + 0x0f, 0xde, 0x53, 0xe8, 0xd7, 0xe3, 0xee, 0x68, 0xf9, 0xe5, 0x09, 0x6d, 0xd7, 0xe6, 0x82, 0xda, + 0x07, 0x37, 0xe3, 0xed, 0x33, 0xd1, 0x64, 0xb4, 0xff, 0x08, 0x66, 0xc4, 0x6d, 0x0b, 0xad, 0x9d, + 0x7e, 0x0d, 0x33, 0xae, 0x54, 0xcf, 0xa7, 0x97, 0x09, 0xce, 0xed, 0xe0, 0xbc, 0xf8, 0x6d, 0xf0, + 0xed, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x8a, 0x85, 0xda, 0x2e, 0x4d, 0x28, 0x00, 0x00, } diff --git a/weed/pb/volume_server_pb/volume_server_helper.go b/weed/pb/volume_server_pb/volume_server_helper.go new file mode 100644 index 000000000..356be27ff --- /dev/null +++ b/weed/pb/volume_server_pb/volume_server_helper.go @@ -0,0 +1,5 @@ +package volume_server_pb + +func (m *RemoteFile) BackendName() string { + return m.BackendType + "." + m.BackendId +} diff --git a/weed/server/common.go b/weed/server/common.go index e3e16b927..ba5562dfe 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -11,15 +11,12 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/storage/needle" - "google.golang.org/grpc" - - _ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/stats" + "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" _ "github.com/chrislusf/seaweedfs/weed/statik" "github.com/gorilla/mux" diff --git a/weed/server/volume_grpc_tier.go b/weed/server/volume_grpc_tier.go index cbd344ed0..d3cdd62f2 100644 --- a/weed/server/volume_grpc_tier.go +++ b/weed/server/volume_grpc_tier.go @@ -1,9 +1,9 @@ package weed_server import ( - "context" "fmt" "os" + "time" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/backend" @@ -11,28 +11,83 @@ import ( ) // VolumeTierCopyDatToRemote copy dat file to a remote tier -func (vs *VolumeServer) VolumeTierCopyDatToRemote(ctx context.Context, req *volume_server_pb.VolumeTierCopyDatToRemoteRequest) (*volume_server_pb.VolumeTierCopyDatToRemoteResponse, error) { +func (vs *VolumeServer) VolumeTierCopyDatToRemote(req *volume_server_pb.VolumeTierCopyDatToRemoteRequest, stream volume_server_pb.VolumeServer_VolumeTierCopyDatToRemoteServer) error { + // find existing volume v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) if v == nil { - return nil, fmt.Errorf("volume %d not found", req.VolumeId) + return fmt.Errorf("volume %d not found", req.VolumeId) } + // verify the collection if v.Collection != req.Collection { - return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection) + return fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection) } + // locate the disk file diskFile, ok := v.DataBackend.(*backend.DiskFile) if !ok { - return nil, fmt.Errorf("volume %d is not on local disk", req.VolumeId) + return fmt.Errorf("volume %d is not on local disk", req.VolumeId) } - err := uploadFileToRemote(ctx, req, diskFile.File) - return &volume_server_pb.VolumeTierCopyDatToRemoteResponse{}, err -} + // check valid storage backend type + backendStorage, found := backend.BackendStorages[req.DestinationBackendName] + if !found { + var keys []string + for key := range backend.BackendStorages { + keys = append(keys, key) + } + return fmt.Errorf("destination %s not found, suppported: %v", req.DestinationBackendName, keys) + } -func uploadFileToRemote(ctx context.Context, req *volume_server_pb.VolumeTierCopyDatToRemoteRequest, f *os.File) error { - println("copying dat file of", f.Name(), "to remote") + // check whether the existing backend storage is the same as requested + // if same, skip + backendType, backendId := backend.BackendNameToTypeId(req.DestinationBackendName) + for _, remoteFile := range v.GetVolumeTierInfo().GetFiles() { + if remoteFile.BackendType == backendType && remoteFile.BackendId == backendId { + return fmt.Errorf("destination %s already exists", req.DestinationBackendName) + } + } + + startTime := time.Now() + fn := func(progressed int64, percentage float32) error { + now := time.Now() + if now.Sub(startTime) < time.Second { + return nil + } + startTime = now + return stream.Send(&volume_server_pb.VolumeTierCopyDatToRemoteResponse{ + Processed: progressed, + ProcessedPercentage: percentage, + }) + } + // copy the data file + key, size, err := backendStorage.CopyFile(diskFile.File, fn) + if err != nil { + return fmt.Errorf("backend %s copy file %s: %v", req.DestinationBackendName, diskFile.String(), err) + } + + // save the remote file to volume tier info + v.GetVolumeTierInfo().Files = append(v.GetVolumeTierInfo().GetFiles(), &volume_server_pb.RemoteFile{ + BackendType: backendType, + BackendId: backendId, + Key: key, + Offset: 0, + FileSize: uint64(size), + ModifiedTime: uint64(time.Now().Unix()), + }) + + if err := v.SaveVolumeTierInfo(); err != nil { + return fmt.Errorf("volume %d fail to save remote file info: %v", v.Id, err) + } + + if err := v.LoadRemoteFile(); err != nil { + return fmt.Errorf("volume %d fail to load remote file: %v", v.Id, err) + } + + if !req.KeepLocalDatFile { + os.Remove(v.FileName() + ".dat") + } return nil } diff --git a/weed/shell/command_volume_tier.go b/weed/shell/command_volume_tier.go index 177d3f30d..9047d1199 100644 --- a/weed/shell/command_volume_tier.go +++ b/weed/shell/command_volume_tier.go @@ -25,14 +25,30 @@ func (c *commandVolumeTier) Name() string { } func (c *commandVolumeTier) Help() string { - return `copy the dat file of a volume to a remote tier + return `move the dat file of a volume to a remote tier - ec.encode [-collection=""] [-fullPercent=95] [-quietFor=1h] - ec.encode [-collection=""] [-volumeId=] + volume.tier [-collection=""] [-fullPercent=95] [-quietFor=1h] + volume.tier [-collection=""] -volumeId= -dest= [-keepLocalDatFile] - This command will: - 1. freeze one volume - 2. copy the dat file of a volume to a remote tier + e.g.: + volume.tier -volumeId=7 -dest=s3 + volume.tier -volumeId=7 -dest=s3.default + + The is defined in master.toml. + For example, "s3.default" in [storage.backend.s3.default] + + This command will move the dat file of a volume to a remote tier. + + SeaweedFS enables scalable and fast local access to lots of files, + and the cloud storage is slower by cost efficient. How to combine them together? + + Usually the data follows 80/20 rule: only 20% of data is frequently accessed. + We can offload the old volumes to the cloud. + + With this, SeaweedFS can be both fast and scalable, and infinite storage space. + Just add more local SeaweedFS volume servers to increase the throughput. + + The index file is still local, and the same O(1) disk read is applied to the remote file. ` } @@ -44,7 +60,8 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io. collection := tierCommand.String("collection", "", "the collection name") fullPercentage := tierCommand.Float64("fullPercent", 95, "the volume reaches the percentage of max volume size") quietPeriod := tierCommand.Duration("quietFor", 24*time.Hour, "select volumes without no writes for this period") - dest := tierCommand.String("destination", "", "the target tier name") + dest := tierCommand.String("dest", "", "the target tier name") + keepLocalDatFile := tierCommand.Bool("keepLocalDatFile", false, "whether keep local dat file") if err = tierCommand.Parse(args); err != nil { return nil } @@ -54,7 +71,7 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io. // volumeId is provided if vid != 0 { - return doVolumeTier(ctx, commandEnv, *collection, vid, *dest) + return doVolumeTier(ctx, commandEnv, writer, *collection, vid, *dest, *keepLocalDatFile) } // apply to all volumes in the collection @@ -65,7 +82,7 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io. } fmt.Printf("tiering volumes: %v\n", volumeIds) for _, vid := range volumeIds { - if err = doVolumeTier(ctx, commandEnv, *collection, vid, *dest); err != nil { + if err = doVolumeTier(ctx, commandEnv, writer, *collection, vid, *dest, *keepLocalDatFile); err != nil { return err } } @@ -73,7 +90,7 @@ func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, collection string, vid needle.VolumeId, dest string) (err error) { +func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, writer io.Writer, collection string, vid needle.VolumeId, dest string, keepLocalDatFile bool) (err error) { // find volume location locations, found := commandEnv.MasterClient.GetLocations(uint32(vid)) if !found { @@ -89,7 +106,7 @@ func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, collection string */ // copy the .dat file to remote tier - err = copyDatToRemoteTier(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(vid), collection, locations[0].Url, dest) + err = copyDatToRemoteTier(ctx, commandEnv.option.GrpcDialOption, writer, needle.VolumeId(vid), collection, locations[0].Url, dest, keepLocalDatFile) if err != nil { return fmt.Errorf("copy dat file for volume %d on %s to %s: %v", vid, locations[0].Url, dest, err) } @@ -97,13 +114,34 @@ func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, collection string return nil } -func copyDatToRemoteTier(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer string, dest string) error { +func copyDatToRemoteTier(ctx context.Context, grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, collection string, sourceVolumeServer string, dest string, keepLocalDatFile bool) error { err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, copyErr := volumeServerClient.VolumeTierCopyDatToRemote(ctx, &volume_server_pb.VolumeTierCopyDatToRemoteRequest{ - VolumeId: uint32(volumeId), - Collection: collection, + stream, copyErr := volumeServerClient.VolumeTierCopyDatToRemote(ctx, &volume_server_pb.VolumeTierCopyDatToRemoteRequest{ + VolumeId: uint32(volumeId), + Collection: collection, + DestinationBackendName: dest, + KeepLocalDatFile: keepLocalDatFile, }) + + var lastProcessed int64 + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + if recvErr == io.EOF { + break + } else { + return recvErr + } + } + + processingSpeed := float64(resp.Processed - lastProcessed)/1024.0/1024.0 + + fmt.Fprintf(writer, "copied %.2f%%, %d bytes, %.2fMB/s\n", resp.ProcessedPercentage, resp.Processed, processingSpeed) + + lastProcessed = resp.Processed + } + return copyErr }) diff --git a/weed/storage/backend/backend.go b/weed/storage/backend/backend.go index 2552e5c06..4b54d89dd 100644 --- a/weed/storage/backend/backend.go +++ b/weed/storage/backend/backend.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/spf13/viper" ) @@ -23,7 +24,8 @@ type BackendStorageFile interface { type BackendStorage interface { ToProperties() map[string]string - NewStorageFile(key string) BackendStorageFile + NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) BackendStorageFile + CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) } type StringProperties interface { @@ -46,13 +48,13 @@ func LoadConfiguration(config *viper.Viper) { backendSub := config.Sub(StorageBackendPrefix) - for backendTypeName, _ := range config.GetStringMap(StorageBackendPrefix) { + for backendTypeName := range config.GetStringMap(StorageBackendPrefix) { backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)] if !found { glog.Fatalf("backend storage type %s not found", backendTypeName) } backendTypeSub := backendSub.Sub(backendTypeName) - for backendStorageId, _ := range backendSub.GetStringMap(backendTypeName) { + for backendStorageId := range backendSub.GetStringMap(backendTypeName) { if !backendTypeSub.GetBool(backendStorageId + ".enabled") { continue } @@ -105,12 +107,10 @@ func (p *Properties) GetString(key string) string { func ToPbStorageBackends() (backends []*master_pb.StorageBackend) { for sName, s := range BackendStorages { - parts := strings.Split(sName, ".") - if len(parts) != 2 { + sType, sId := BackendNameToTypeId(sName) + if sType == "" { continue } - - sType, sId := parts[0], parts[1] backends = append(backends, &master_pb.StorageBackend{ Type: sType, Id: sId, @@ -119,3 +119,16 @@ func ToPbStorageBackends() (backends []*master_pb.StorageBackend) { } return } + +func BackendNameToTypeId(backendName string) (backendType, backendId string) { + parts := strings.Split(backendName, ".") + if len(parts) == 1 { + return backendName, "default" + } + if len(parts) != 2 { + return + } + + backendType, backendId = parts[0], parts[1] + return +} diff --git a/weed/storage/backend/s3_backend/s3_backend.go b/weed/storage/backend/s3_backend/s3_backend.go index 7db8f2337..66299be48 100644 --- a/weed/storage/backend/s3_backend/s3_backend.go +++ b/weed/storage/backend/s3_backend/s3_backend.go @@ -2,6 +2,7 @@ package s3_backend import ( "fmt" + "io" "os" "strings" "time" @@ -9,7 +10,9 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/backend" + "github.com/google/uuid" ) func init() { @@ -57,7 +60,7 @@ func (s *S3BackendStorage) ToProperties() map[string]string { return m } -func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile { +func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) backend.BackendStorageFile { if strings.HasPrefix(key, "/") { key = key[1:] } @@ -65,18 +68,35 @@ func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile f := &S3BackendStorageFile{ backendStorage: s, key: key, + tierInfo: tierInfo, } return f } +func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) { + randomUuid, _ := uuid.NewRandom() + key = randomUuid.String() + + glog.V(1).Infof("copying dat file of", f.Name(), "to remote s3", s.id, "as", key) + + size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, fn) + + return +} + type S3BackendStorageFile struct { backendStorage *S3BackendStorage key string + tierInfo *volume_server_pb.VolumeTierInfo } func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) { + bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1) + + // glog.V(0).Infof("read %s %s", s3backendStorageFile.key, bytesRange) + getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{ Bucket: &s3backendStorageFile.backendStorage.bucket, Key: &s3backendStorageFile.key, @@ -84,13 +104,26 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n }) if getObjectErr != nil { - return 0, fmt.Errorf("bucket %s GetObject %s: %v", - s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr) + return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr) } defer getObjectOutput.Body.Close() - return getObjectOutput.Body.Read(p) + glog.V(4).Infof("read %s %s", s3backendStorageFile.key, bytesRange) + glog.V(4).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength) + for { + if n, err = getObjectOutput.Body.Read(p); err == nil && n < len(p) { + p = p[n:] + } else { + break + } + } + + if err == io.EOF { + err = nil + } + + return } func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) { @@ -107,18 +140,15 @@ func (s3backendStorageFile S3BackendStorageFile) Close() error { func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) { - headObjectOutput, headObjectErr := s3backendStorageFile.backendStorage.conn.HeadObject(&s3.HeadObjectInput{ - Bucket: &s3backendStorageFile.backendStorage.bucket, - Key: &s3backendStorageFile.key, - }) + files := s3backendStorageFile.tierInfo.GetFiles() - if headObjectErr != nil { - return 0, time.Now(), fmt.Errorf("bucket %s HeadObject %s: %v", - s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, headObjectErr) + if len(files)==0 { + err = fmt.Errorf("remote file info not found") + return } - datSize = int64(*headObjectOutput.ContentLength) - modTime = *headObjectOutput.LastModified + datSize = int64(files[0].FileSize) + modTime = time.Unix(int64(files[0].ModifiedTime),0) return } diff --git a/weed/storage/backend/s3_backend/s3_upload.go b/weed/storage/backend/s3_backend/s3_upload.go index 1fde0a84f..38acd18ef 100644 --- a/weed/storage/backend/s3_backend/s3_upload.go +++ b/weed/storage/backend/s3_backend/s3_upload.go @@ -3,27 +3,30 @@ package s3_backend import ( "fmt" "os" + "sync/atomic" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/chrislusf/seaweedfs/weed/glog" ) -func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string) error { +func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string, + fn func(progressed int64, percentage float32)error) (fileSize int64, err error) { //open the file f, err := os.Open(filename) if err != nil { - return fmt.Errorf("failed to open file %q, %v", filename, err) + return 0, fmt.Errorf("failed to open file %q, %v", filename, err) } defer f.Close() info, err := f.Stat() if err != nil { - return fmt.Errorf("failed to stat file %q, %v", filename, err) + return 0, fmt.Errorf("failed to stat file %q, %v", filename, err) } - fileSize := info.Size() + fileSize = info.Size() partSize := int64(64 * 1024 * 1024) // The minimum/default allowed part size is 5MB for partSize*1000 < fileSize { @@ -33,14 +36,22 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey // Create an uploader with the session and custom options uploader := s3manager.NewUploaderWithClient(sess, func(u *s3manager.Uploader) { u.PartSize = partSize - u.Concurrency = 15 // default is 15 + u.Concurrency = 5 }) + fileReader := &s3UploadProgressedReader{ + fp: f, + size:fileSize, + read:-fileSize, + fn:fn, + } + // Upload the file to S3. - result, err := uploader.Upload(&s3manager.UploadInput{ + var result *s3manager.UploadOutput + result, err = uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(destBucket), Key: aws.String(destKey), - Body: f, + Body: fileReader, ACL: aws.String("private"), ServerSideEncryption: aws.String("AES256"), StorageClass: aws.String("STANDARD_IA"), @@ -48,9 +59,44 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey //in case it fails to upload if err != nil { - return fmt.Errorf("failed to upload file, %v", err) + return 0, fmt.Errorf("failed to upload file %s: %v", filename, err) } - fmt.Printf("file %s uploaded to %s\n", filename, result.Location) + glog.V(1).Infof("file %s uploaded to %s\n", filename, result.Location) - return nil + return +} + +// adapted from https://github.com/aws/aws-sdk-go/pull/1868 +type s3UploadProgressedReader struct { + fp *os.File + size int64 + read int64 + fn func(progressed int64, percentage float32)error +} +func (r *s3UploadProgressedReader) Read(p []byte) (int, error) { + return r.fp.Read(p) +} + +func (r *s3UploadProgressedReader) ReadAt(p []byte, off int64) (int, error) { + n, err := r.fp.ReadAt(p, off) + if err != nil { + return n, err + } + + // Got the length have read( or means has uploaded), and you can construct your message + atomic.AddInt64(&r.read, int64(n)) + + if r.fn != nil { + read := r.read + if err := r.fn(read, float32(read*100)/float32(r.size)); err != nil { + return n, err + } + } + + return n, err +} + + +func (r *s3UploadProgressedReader) Seek(offset int64, whence int) (int64, error) { + return r.fp.Seek(offset, whence) } diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index c7faa57a6..66d3b86c2 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -33,8 +33,8 @@ func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation { func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) { name := dir.Name() - if !dir.IsDir() && strings.HasSuffix(name, ".dat") { - base := name[:len(name)-len(".dat")] + if !dir.IsDir() && strings.HasSuffix(name, ".idx") { + base := name[:len(name)-len(".idx")] collection, volumeId, err := parseCollectionVolumeId(base) return volumeId, collection, err } @@ -53,7 +53,7 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) { name := fileInfo.Name() - if !fileInfo.IsDir() && strings.HasSuffix(name, ".dat") { + if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") { vid, collection, err := l.volumeIdFromPath(fileInfo) if err == nil { l.RLock() diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 00d76a4b3..ee05126e1 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -37,6 +38,8 @@ type Volume struct { lastCompactRevision uint16 isCompacting bool + + volumeTierInfo *volume_server_pb.VolumeTierInfo } func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) { diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index 6f1d8fe40..e09f848ab 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -26,8 +26,11 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind fileName := v.FileName() alreadyHasSuperBlock := false - // open dat file - if exists, canRead, canWrite, modifiedTime, fileSize := checkFile(fileName + ".dat"); exists { + if v.maybeLoadVolumeTierInfo() { + // open remote file + alreadyHasSuperBlock = true + } else if exists, canRead, canWrite, modifiedTime, fileSize := checkFile(fileName + ".dat"); exists { + // open dat file if !canRead { return fmt.Errorf("cannot read Volume Data file %s.dat", fileName) } diff --git a/weed/storage/volume_tier.go b/weed/storage/volume_tier.go new file mode 100644 index 000000000..d469d7483 --- /dev/null +++ b/weed/storage/volume_tier.go @@ -0,0 +1,104 @@ +package storage + +import ( + "bytes" + "fmt" + "io/ioutil" + + _ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage/backend" + "github.com/golang/protobuf/jsonpb" +) + +func (v *Volume) GetVolumeTierInfo() *volume_server_pb.VolumeTierInfo { + return v.volumeTierInfo +} + +func (v *Volume) maybeLoadVolumeTierInfo() bool { + + v.volumeTierInfo = &volume_server_pb.VolumeTierInfo{} + + tierFileName := v.FileName() + ".tier" + + if exists, canRead, _, _, _ := checkFile(tierFileName); !exists || !canRead { + if !exists { + return false + } + if !canRead { + glog.Warningf("can not read %s", tierFileName) + } + return false + } + + glog.V(0).Infof("maybeLoadVolumeTierInfo loading volume %d check file", v.Id) + + tierData, readErr := ioutil.ReadFile(tierFileName) + if readErr != nil { + glog.Warningf("fail to read %s : %v", tierFileName, readErr) + return false + } + + glog.V(0).Infof("maybeLoadVolumeTierInfo loading volume %d ReadFile", v.Id) + + if err := jsonpb.Unmarshal(bytes.NewReader(tierData), v.volumeTierInfo); err != nil { + glog.Warningf("unmarshal error: %v", err) + return false + } + + glog.V(0).Infof("maybeLoadVolumeTierInfo loading volume %d Unmarshal tierInfo %v", v.Id, v.volumeTierInfo) + + if len(v.volumeTierInfo.GetFiles()) == 0 { + return false + } + + glog.V(0).Infof("volume %d is tiered to %s as %s and read only", v.Id, + v.volumeTierInfo.Files[0].BackendName(), v.volumeTierInfo.Files[0].Key) + + v.readOnly = true + + glog.V(0).Infof("loading volume %d from remote %v", v.Id, v.volumeTierInfo.Files) + v.LoadRemoteFile() + + return true +} + +func (v *Volume) LoadRemoteFile() error { + tierFile := v.volumeTierInfo.GetFiles()[0] + backendStorage := backend.BackendStorages[tierFile.BackendName()] + + if v.DataBackend != nil { + v.DataBackend.Close() + } + + v.DataBackend = backendStorage.NewStorageFile(tierFile.Key, v.volumeTierInfo) + return nil +} + +func (v *Volume) SaveVolumeTierInfo() error { + + tierFileName := v.FileName() + ".tier" + + if exists, _, canWrite, _, _ := checkFile(tierFileName); exists && !canWrite { + return fmt.Errorf("%s not writable", tierFileName) + } + + m := jsonpb.Marshaler{ + EmitDefaults: true, + Indent: " ", + } + + text, marshalErr := m.MarshalToString(v.GetVolumeTierInfo()) + if marshalErr != nil { + return fmt.Errorf("marshal volume %d tier info: %v", v.Id, marshalErr) + } + + writeErr := ioutil.WriteFile(tierFileName, []byte(text), 0755) + if writeErr != nil { + return fmt.Errorf("fail to write %s : %v", tierFileName, writeErr) + } + + return nil +}