diff --git a/weed/pb/filer_pb/filer_grpc.pb.go b/weed/pb/filer_pb/filer_grpc.pb.go index 4a5f47d71..c5ebc70c4 100644 --- a/weed/pb/filer_pb/filer_grpc.pb.go +++ b/weed/pb/filer_pb/filer_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.17.3 +// source: filer.proto package filer_pb diff --git a/weed/pb/iam_pb/iam_grpc.pb.go b/weed/pb/iam_pb/iam_grpc.pb.go index b9438a295..704bf5c7c 100644 --- a/weed/pb/iam_pb/iam_grpc.pb.go +++ b/weed/pb/iam_pb/iam_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.17.3 +// source: iam.proto package iam_pb diff --git a/weed/pb/master.proto b/weed/pb/master.proto index 751555eb7..a81497bb6 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -70,7 +70,7 @@ message Heartbeat { map max_volume_counts = 4; uint32 grpc_port = 20; - + repeated string LocationUUIDs = 21; } message HeartbeatResponse { @@ -79,6 +79,7 @@ message HeartbeatResponse { string metrics_address = 3; uint32 metrics_interval_seconds = 4; repeated StorageBackend storage_backends = 5; + bool has_duplicated_directory = 6; } message VolumeInformationMessage { diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 53e268920..d0b821c98 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -45,6 +45,7 @@ type Heartbeat struct { HasNoEcShards bool `protobuf:"varint,19,opt,name=has_no_ec_shards,json=hasNoEcShards,proto3" json:"has_no_ec_shards,omitempty"` MaxVolumeCounts map[string]uint32 `protobuf:"bytes,4,rep,name=max_volume_counts,json=maxVolumeCounts,proto3" json:"max_volume_counts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` GrpcPort uint32 `protobuf:"varint,20,opt,name=grpc_port,json=grpcPort,proto3" json:"grpc_port,omitempty"` + LocationUUIDs []string `protobuf:"bytes,21,rep,name=LocationUUIDs,proto3" json:"LocationUUIDs,omitempty"` } func (x *Heartbeat) Reset() { @@ -198,6 +199,13 @@ func (x *Heartbeat) GetGrpcPort() uint32 { return 0 } +func (x *Heartbeat) GetLocationUUIDs() []string { + if x != nil { + return x.LocationUUIDs + } + return nil +} + type HeartbeatResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -208,6 +216,7 @@ type HeartbeatResponse struct { MetricsAddress string `protobuf:"bytes,3,opt,name=metrics_address,json=metricsAddress,proto3" json:"metrics_address,omitempty"` MetricsIntervalSeconds uint32 `protobuf:"varint,4,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds,proto3" json:"metrics_interval_seconds,omitempty"` StorageBackends []*StorageBackend `protobuf:"bytes,5,rep,name=storage_backends,json=storageBackends,proto3" json:"storage_backends,omitempty"` + HasDuplicatedDirectory bool `protobuf:"varint,6,opt,name=has_duplicated_directory,json=hasDuplicatedDirectory,proto3" json:"has_duplicated_directory,omitempty"` } func (x *HeartbeatResponse) Reset() { @@ -277,6 +286,13 @@ func (x *HeartbeatResponse) GetStorageBackends() []*StorageBackend { return nil } +func (x *HeartbeatResponse) GetHasDuplicatedDirectory() bool { + if x != nil { + return x.HasDuplicatedDirectory + } + return false +} + type VolumeInformationMessage struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3640,7 +3656,7 @@ var File_master_proto protoreflect.FileDescriptor var file_master_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, - 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x22, 0x99, 0x07, 0x0a, 0x09, 0x48, 0x65, + 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x22, 0xbf, 0x07, 0x0a, 0x09, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x70, @@ -3694,27 +3710,33 @@ var file_master_proto_rawDesc = []byte{ 0x74, 0x72, 0x79, 0x52, 0x0f, 0x6d, 0x61, 0x78, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, - 0x74, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x61, 0x78, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x80, 0x02, 0x0a, 0x11, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, - 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x69, - 0x7a, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, - 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, - 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x16, 0x6d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x73, 0x12, 0x44, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, - 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, - 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x22, 0x98, 0x04, 0x0a, 0x18, 0x56, 0x6f, 0x6c, + 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x55, 0x49, + 0x44, 0x73, 0x18, 0x15, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x55, 0x55, 0x49, 0x44, 0x73, 0x1a, 0x42, 0x0a, 0x14, 0x4d, 0x61, 0x78, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xba, 0x02, 0x0a, 0x11, + 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, + 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x76, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x38, + 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, + 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x16, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, + 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x44, 0x0a, 0x10, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x0f, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x12, 0x38, + 0x0a, 0x18, 0x68, 0x61, 0x73, 0x5f, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, + 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x16, 0x68, 0x61, 0x73, 0x44, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x44, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x22, 0x98, 0x04, 0x0a, 0x18, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, diff --git a/weed/pb/master_pb/master_grpc.pb.go b/weed/pb/master_pb/master_grpc.pb.go index f8b11e8c5..b1be34957 100644 --- a/weed/pb/master_pb/master_grpc.pb.go +++ b/weed/pb/master_pb/master_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.17.3 +// source: master.proto package master_pb diff --git a/weed/pb/messaging_pb/messaging_grpc.pb.go b/weed/pb/messaging_pb/messaging_grpc.pb.go index 234cffa95..6fd396f55 100644 --- a/weed/pb/messaging_pb/messaging_grpc.pb.go +++ b/weed/pb/messaging_pb/messaging_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.17.3 +// source: messaging.proto package messaging_pb diff --git a/weed/pb/mount_pb/mount_grpc.pb.go b/weed/pb/mount_pb/mount_grpc.pb.go index 41737aa21..98d290cd3 100644 --- a/weed/pb/mount_pb/mount_grpc.pb.go +++ b/weed/pb/mount_pb/mount_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.17.3 +// source: mount.proto package mount_pb diff --git a/weed/pb/volume_server_pb/volume_server_grpc.pb.go b/weed/pb/volume_server_pb/volume_server_grpc.pb.go index e1b162457..2669c75e4 100644 --- a/weed/pb/volume_server_pb/volume_server_grpc.pb.go +++ b/weed/pb/volume_server_pb/volume_server_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.17.3 +// source: volume_server.proto package volume_server_pb diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index 981f663e4..b2ac4e700 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -2,12 +2,16 @@ package weed_server import ( "context" + "errors" + "fmt" + "net" + "sort" + "time" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/util" - "net" - "time" "github.com/chrislusf/raft" "google.golang.org/grpc/peer" @@ -18,6 +22,37 @@ import ( "github.com/chrislusf/seaweedfs/weed/topology" ) +func (ms *MasterServer) RegisterUUIDs(heartbeat *master_pb.Heartbeat) error { + ms.Topo.UUIDAccessLock.Lock() + defer ms.Topo.UUIDAccessLock.Unlock() + key := fmt.Sprintf("%s:%d", heartbeat.Ip, heartbeat.Port) + if ms.Topo.UUIDMap == nil { + ms.Topo.UUIDMap = make(map[string][]string) + } + // find whether new UUID exists + for k, v := range ms.Topo.UUIDMap { + for _, id := range heartbeat.LocationUUIDs { + sort.Strings(v) + index := sort.SearchStrings(v, id) + if index < len(v) && v[index] == id { + glog.Error("directory of ", id, " on ", k, " has been loaded") + return errors.New("volume: Duplicated volume directory was been loaded") + } + } + } + ms.Topo.UUIDMap[key] = heartbeat.LocationUUIDs + glog.V(0).Infof("found new UUID:%v %v , %v", key, heartbeat.LocationUUIDs, ms.Topo.UUIDMap) + return nil +} + +func (ms *MasterServer) UnRegisterUUIDs(ip string, port int) { + ms.Topo.UUIDAccessLock.Lock() + defer ms.Topo.UUIDAccessLock.Unlock() + key := fmt.Sprintf("%s:%d", ip, port) + delete(ms.Topo.UUIDMap, key) + glog.V(0).Infof("remove volume server %v, online volume server: %v", key, ms.Topo.UUIDMap) +} + func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error { var dn *topology.DataNode @@ -32,6 +67,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ // the unregister and register can race with each other ms.Topo.UnRegisterDataNode(dn) glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port) + ms.UnRegisterUUIDs(dn.Ip, dn.Port) message := &master_pb.VolumeLocation{ Url: dn.Url(), @@ -69,7 +105,18 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ dc := ms.Topo.GetOrCreateDataCenter(dcName) rack := dc.GetOrCreateRack(rackName) dn = rack.GetOrCreateDataNode(heartbeat.Ip, int(heartbeat.Port), int(heartbeat.GrpcPort), heartbeat.PublicUrl, heartbeat.MaxVolumeCounts) - glog.V(0).Infof("added volume server %d: %v:%d", dn.Counter, heartbeat.GetIp(), heartbeat.GetPort()) + glog.V(0).Infof("added volume server %d: %v:%d %v", dn.Counter, heartbeat.GetIp(), heartbeat.GetPort(), heartbeat.LocationUUIDs) + err := ms.RegisterUUIDs(heartbeat) + if err != nil { + if stream_err := stream.Send(&master_pb.HeartbeatResponse{ + HasDuplicatedDirectory: true, + }); stream_err != nil { + glog.Warningf("SendHeartbeat.Send DuplicatedDirectory response to %s:%d %v", dn.Ip, dn.Port, stream_err) + return stream_err + } + return err + } + if err := stream.Send(&master_pb.HeartbeatResponse{ VolumeSizeLimit: uint64(ms.option.VolumeSizeLimitMB) * 1024 * 1024, }); err != nil { diff --git a/weed/server/volume_grpc_client_to_master.go b/weed/server/volume_grpc_client_to_master.go index f3f99ee7b..5e341f07c 100644 --- a/weed/server/volume_grpc_client_to_master.go +++ b/weed/server/volume_grpc_client_to_master.go @@ -2,9 +2,12 @@ package weed_server import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/operation" + "os" + "syscall" "time" + "github.com/chrislusf/seaweedfs/weed/operation" + "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb" @@ -116,6 +119,12 @@ func (vs *VolumeServer) doHeartbeat(masterAddress pb.ServerAddress, grpcDialOpti doneChan <- err return } + if in.HasDuplicatedDirectory { + glog.Error("Shut Down Volume Server due to duplicated volume directory") + glog.V(0).Infof("send SIGINT to Volume Server") + p, _ := os.FindProcess(vs.pid) + p.Signal(syscall.SIGINT) + } if in.GetVolumeSizeLimit() != 0 && vs.store.GetVolumeSizeLimit() != in.GetVolumeSizeLimit() { vs.store.SetVolumeSizeLimit(in.GetVolumeSizeLimit()) if vs.store.MaybeAdjustVolumeMax() { diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index dcd27673c..f927dbdb8 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -1,11 +1,13 @@ package weed_server import ( + "net/http" + "os" + "sync" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/types" - "net/http" - "sync" "google.golang.org/grpc" @@ -43,6 +45,7 @@ type VolumeServer struct { fileSizeLimitBytes int64 isHeartbeating bool stopChan chan bool + pid int } func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, @@ -86,6 +89,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), concurrentUploadLimit: concurrentUploadLimit, concurrentDownloadLimit: concurrentDownloadLimit, + pid: os.Getpid(), } vs.SeedMasterNodes = masterNodes diff --git a/weed/server/volume_server_tcp_handlers_write.go b/weed/server/volume_server_tcp_handlers_write.go index a009611da..24ad916e6 100644 --- a/weed/server/volume_server_tcp_handlers_write.go +++ b/weed/server/volume_server_tcp_handlers_write.go @@ -3,12 +3,13 @@ package weed_server import ( "bufio" "fmt" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/storage/needle" - "github.com/chrislusf/seaweedfs/weed/util" "io" "net" "strings" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/util" ) func (vs *VolumeServer) HandleTcpConnection(c net.Conn) { diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index e92810022..ae2a36296 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -14,6 +14,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/google/uuid" ) type DiskLocation struct { @@ -33,6 +34,29 @@ type DiskLocation struct { isDiskSpaceLow bool } +func GenerateDirUUID(dir string) (dirUUIDString string, err error) { + glog.V(1).Infof("Getting UUID of volume directory:%s", dir) + dirUUIDString = "" + fileName := dir + "/volume.uuid" + if !util.FileExists(fileName) { + dirUUID, _ := uuid.NewRandom() + dirUUIDString = dirUUID.String() + writeErr := util.WriteFile(fileName, []byte(dirUUIDString), 0644) + if writeErr != nil { + glog.Warningf("failed to write UUID to %s : %v", fileName, writeErr) + return "", fmt.Errorf("failed to write UUID to %s : %v", fileName, writeErr) + } + } else { + uuidData, readErr := os.ReadFile(fileName) + if readErr != nil { + glog.Warningf("failed to read UUID from %s : %v", fileName, readErr) + return "", fmt.Errorf("failed to read UUID from %s : %v", fileName, readErr) + } + dirUUIDString = string(uuidData) + } + return dirUUIDString, nil +} + func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation { dir = util.ResolvePath(dir) if idxDir == "" { diff --git a/weed/storage/store.go b/weed/storage/store.go index fa2897fbc..2df0aee83 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -2,13 +2,14 @@ package storage import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/storage/volume_info" - "github.com/chrislusf/seaweedfs/weed/util" "path/filepath" "strings" "sync/atomic" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/storage/volume_info" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/glog" @@ -40,6 +41,7 @@ type Store struct { GrpcPort int PublicUrl string Locations []*DiskLocation + LocationUUIDs []string dataCenter string // optional informaton, overwriting master setting if exists rack string // optional information, overwriting master setting if exists connected bool @@ -64,6 +66,8 @@ func NewStore(grpcDialOption grpc.DialOption, ip string, port int, grpcPort int, location := NewDiskLocation(dirnames[i], maxVolumeCounts[i], minFreeSpaces[i], idxFolder, diskTypes[i]) location.loadExistingVolumes(needleMapKind) s.Locations = append(s.Locations, location) + dirUUID, _ := GenerateDirUUID(dirnames[i]) + s.LocationUUIDs = append(s.LocationUUIDs, dirUUID) stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i])) } s.NewVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3) @@ -300,6 +304,11 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { } } + var UUIDList []string + for _, locationUUID := range s.LocationUUIDs { + UUIDList = append(UUIDList, locationUUID) + } + for col, size := range collectionVolumeSize { stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size)) } @@ -321,6 +330,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { Rack: s.rack, Volumes: volumeMessages, HasNoVolumes: len(volumeMessages) == 0, + LocationUUIDs: UUIDList, } } diff --git a/weed/storage/volume_stream_write.go b/weed/storage/volume_stream_write.go index d229bdf20..bd8084a3b 100644 --- a/weed/storage/volume_stream_write.go +++ b/weed/storage/volume_stream_write.go @@ -3,10 +3,11 @@ package storage import ( "bufio" "fmt" - "github.com/chrislusf/seaweedfs/weed/util" "io" "time" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/storage/needle" diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 6d5b373e9..f82f63379 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -4,12 +4,13 @@ import ( "encoding/json" "errors" "fmt" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/storage/types" "math/rand" "sync" "time" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/raft" hashicorpRaft "github.com/hashicorp/raft" @@ -42,8 +43,10 @@ type Topology struct { Configuration *Configuration - RaftServer raft.Server - HashicorpRaft *hashicorpRaft.Raft + RaftServer raft.Server + HashicorpRaft *hashicorpRaft.Raft + UUIDAccessLock sync.RWMutex + UUIDMap map[string][]string } func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology {