gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

rename to lookup

Browse source
This commit is contained in:
chrislu 2023-08-27 18:59:04 -07:00
parent c9caf33119
commit dbcba75271
14 changed files with 432 additions and 515 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
weed/mq/broker/broker_grpc_admin.go
View file

@ -90,36 +90,36 @@ func (broker *MessageQueueBroker) CheckBrokerLoad(c context.Context, request *mq
// createOrUpdateTopicPartitions creates the topic partitions on the broker // createOrUpdateTopicPartitions creates the topic partitions on the broker
// 1. check // 1. check
func (broker *MessageQueueBroker) createOrUpdateTopicPartitions(topic *topic.Topic, prevAssignment *mq_pb.TopicPartitionsAssignment) (err error) { func (broker *MessageQueueBroker) createOrUpdateTopicPartitions(topic *topic.Topic, prevAssignments []*mq_pb.BrokerPartitionAssignment) (err error) {
// create or update each partition // create or update each partition
if prevAssignment == nil { if prevAssignments == nil {
broker.createOrUpdateTopicPartition(topic, nil) broker.createOrUpdateTopicPartition(topic, nil)
} else { } else {
for _, partitionAssignment := range prevAssignment.BrokerPartitions { for _, brokerPartitionAssignment := range prevAssignments {
broker.createOrUpdateTopicPartition(topic, partitionAssignment) broker.createOrUpdateTopicPartition(topic, brokerPartitionAssignment)
} }
} }
return nil return nil
} }
func (broker *MessageQueueBroker) createOrUpdateTopicPartition(topic *topic.Topic, oldAssignment *mq_pb.BrokerPartitionsAssignment) (newAssignment *mq_pb.BrokerPartitionsAssignment) { func (broker *MessageQueueBroker) createOrUpdateTopicPartition(topic *topic.Topic, oldAssignment *mq_pb.BrokerPartitionAssignment) (newAssignment *mq_pb.BrokerPartitionAssignment) {
shouldCreate := broker.confirmBrokerPartitionAssignment(topic, oldAssignment) shouldCreate := broker.confirmBrokerPartitionAssignment(topic, oldAssignment)
if !shouldCreate { if !shouldCreate {
} }
return return
} }
func (broker *MessageQueueBroker) confirmBrokerPartitionAssignment(topic *topic.Topic, oldAssignment *mq_pb.BrokerPartitionsAssignment) (shouldCreate bool) { func (broker *MessageQueueBroker) confirmBrokerPartitionAssignment(topic *topic.Topic, oldAssignment *mq_pb.BrokerPartitionAssignment) (shouldCreate bool) {
if oldAssignment == nil { if oldAssignment == nil {
return true return true
} }
for _, b := range oldAssignment.FollowerBrokers { for _, b := range oldAssignment.FollowerBrokers {
pb.WithBrokerGrpcClient(false, b, broker.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error { pb.WithBrokerGrpcClient(false, b, broker.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
_, err := client.CheckTopicPartitionsStatus(context.Background(), &mq_pb.CheckTopicPartitionsStatusRequest{ _, err := client.CheckTopicPartitionsStatus(context.Background(), &mq_pb.CheckTopicPartitionsStatusRequest{
Namespace: string(topic.Namespace), Namespace: string(topic.Namespace),
Topic: topic.Name, Topic: topic.Name,
BrokerPartitionsAssignment: oldAssignment, BrokerPartitionAssignment: oldAssignment,
ShouldCancelIfNotMatch: true, ShouldCancelIfNotMatch: true,
}) })
if err != nil { if err != nil {
shouldCreate = true shouldCreate = true

17
weed/mq/broker/broker_grpc_lookup.go
View file

@ -17,8 +17,8 @@ import (
// 2.2 if the topic is found, return the brokers // 2.2 if the topic is found, return the brokers
// //
// 3. unlock the topic // 3. unlock the topic
func (broker *MessageQueueBroker) FindTopicBrokers(c context.Context, request *mq_pb.FindTopicBrokersRequest) (*mq_pb.FindTopicBrokersResponse, error) { func (broker *MessageQueueBroker) FindTopicBrokers(ctx context.Context, request *mq_pb.LookupTopicBrokersRequest) (*mq_pb.LookupTopicBrokersResponse, error) {
ret := &mq_pb.FindTopicBrokersResponse{} ret := &mq_pb.LookupTopicBrokersResponse{}
// TODO lock the topic // TODO lock the topic
// find the topic partitions on the filer // find the topic partitions on the filer
@ -27,6 +27,19 @@ func (broker *MessageQueueBroker) FindTopicBrokers(c context.Context, request *m
// create the topic // create the topic
// if the request is_for_subscribe // if the request is_for_subscribe
// return error not found // return error not found
// t := topic.FromPbTopic(request.Topic)
ret.Topic = request.Topic
ret.BrokerPartitionAssignments = []*mq_pb.BrokerPartitionAssignment{
{
LeaderBroker: "localhost:17777",
FollowerBrokers: []string{"localhost:17777"},
Partition: &mq_pb.Partition{
RingSize: MaxPartitionCount,
RangeStart: 0,
RangeStop: MaxPartitionCount,
},
},
}
return ret, nil return ret, nil
} }

4
weed/mq/broker/broker_grpc_pub.go
View file

@ -120,8 +120,8 @@ func (broker *MessageQueueBroker) AssignTopicPartitions(c context.Context, reque
ret := &mq_pb.AssignTopicPartitionsResponse{} ret := &mq_pb.AssignTopicPartitionsResponse{}
self := pb.ServerAddress(fmt.Sprintf("%s:%d", broker.option.Ip, broker.option.Port)) self := pb.ServerAddress(fmt.Sprintf("%s:%d", broker.option.Ip, broker.option.Port))
for _, partition := range request.TopicPartitionsAssignment.BrokerPartitions { for _, brokerPartition := range request.BrokerPartitionAssignments {
localPartiton := topic.FromPbBrokerPartitionsAssignment(self, partition) localPartiton := topic.FromPbBrokerPartitionAssignment(self, brokerPartition)
broker.localTopicManager.AddTopicPartition( broker.localTopicManager.AddTopicPartition(
topic.FromPbTopic(request.Topic), topic.FromPbTopic(request.Topic),
localPartiton) localPartiton)

10
weed/mq/topic/local_partition.go
View file

@ -27,15 +27,11 @@ func (p LocalPartition) Subscribe(clientName string, startReadTime time.Time, ea
}, eachMessageFn) }, eachMessageFn)
} }
func FromPbBrokerPartitionsAssignment(self pb.ServerAddress, assignment *mq_pb.BrokerPartitionsAssignment) *LocalPartition { func FromPbBrokerPartitionAssignment(self pb.ServerAddress, assignment *mq_pb.BrokerPartitionAssignment) *LocalPartition {
isLeaer := assignment.LeaderBroker == string(self) isLeaer := assignment.LeaderBroker == string(self)
localPartition := &LocalPartition{ localPartition := &LocalPartition{
Partition: Partition{ Partition: FromPbPartition(assignment.Partition),
RangeStart: assignment.PartitionStart, isLeader: isLeaer,
RangeStop: assignment.PartitionStop,
RingSize: PartitionCount,
},
isLeader: isLeaer,
} }
if !isLeaer { if !isLeaer {
return localPartition return localPartition

2
weed/pb/filer_pb/filer.pb.go
View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.30.0 // protoc-gen-go v1.28.1
// protoc v4.23.3 // protoc v4.23.3
// source: filer.proto // source: filer.proto

2
weed/pb/iam_pb/iam.pb.go
View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.30.0 // protoc-gen-go v1.28.1
// protoc v4.23.3 // protoc v4.23.3
// source: iam.proto // source: iam.proto

2
weed/pb/master_pb/master.pb.go
View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.30.0 // protoc-gen-go v1.28.1
// protoc v4.23.3 // protoc v4.23.3
// source: master.proto // source: master.proto

2
weed/pb/mount_pb/mount.pb.go
View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.30.0 // protoc-gen-go v1.28.1
// protoc v4.23.3 // protoc v4.23.3
// source: mount.proto // source: mount.proto

29
weed/pb/mq.proto
View file

@ -21,7 +21,7 @@ service SeaweedMessaging {
} }
// control plane for topic partitions // control plane for topic partitions
rpc FindTopicBrokers (FindTopicBrokersRequest) returns (FindTopicBrokersResponse) { rpc LookupTopicBrokers (LookupTopicBrokersRequest) returns (LookupTopicBrokersResponse) {
} }
// a pub client will call this to get the topic partitions assignment // a pub client will call this to get the topic partitions assignment
rpc RequestTopicPartitions (RequestTopicPartitionsRequest) returns (RequestTopicPartitionsResponse) { rpc RequestTopicPartitions (RequestTopicPartitionsRequest) returns (RequestTopicPartitionsResponse) {
@ -100,23 +100,18 @@ message CheckBrokerLoadResponse {
} }
message FindTopicBrokersRequest { message LookupTopicBrokersRequest {
Topic topic = 1; Topic topic = 1;
bool is_for_publish = 2; bool is_for_publish = 2;
} }
message FindTopicBrokersResponse { message LookupTopicBrokersResponse {
Topic topic = 1; Topic topic = 1;
TopicPartitionsAssignment topic_partitions_assignment = 2; repeated BrokerPartitionAssignment broker_partition_assignments = 2;
} }
message BrokerPartitionsAssignment { message BrokerPartitionAssignment {
int32 partition_start = 1; Partition partition = 1;
int32 partition_stop = 2; string leader_broker = 2;
string leader_broker = 3; repeated string follower_brokers = 3;
repeated string follower_brokers = 4;
}
message TopicPartitionsAssignment {
int32 partition_count = 1; // over-sharded partitions, usually 1024
repeated BrokerPartitionsAssignment broker_partitions = 2;
} }
message RequestTopicPartitionsRequest { message RequestTopicPartitionsRequest {
@ -124,12 +119,12 @@ message RequestTopicPartitionsRequest {
int32 partition_count = 2; int32 partition_count = 2;
} }
message RequestTopicPartitionsResponse { message RequestTopicPartitionsResponse {
TopicPartitionsAssignment topic_partitions_assignment = 1; repeated BrokerPartitionAssignment broker_partition_assignments = 1;
} }
message AssignTopicPartitionsRequest { message AssignTopicPartitionsRequest {
Topic topic = 1; Topic topic = 1;
TopicPartitionsAssignment topic_partitions_assignment = 2; repeated BrokerPartitionAssignment broker_partition_assignments = 2;
bool is_leader = 3; bool is_leader = 3;
} }
message AssignTopicPartitionsResponse { message AssignTopicPartitionsResponse {
@ -138,11 +133,11 @@ message AssignTopicPartitionsResponse {
message CheckTopicPartitionsStatusRequest { message CheckTopicPartitionsStatusRequest {
string namespace = 1; string namespace = 1;
string topic = 2; string topic = 2;
BrokerPartitionsAssignment broker_partitions_assignment = 3; BrokerPartitionAssignment broker_partition_assignment = 3;
bool should_cancel_if_not_match = 4; bool should_cancel_if_not_match = 4;
} }
message CheckTopicPartitionsStatusResponse { message CheckTopicPartitionsStatusResponse {
TopicPartitionsAssignment topic_partitions_assignment = 1; repeated BrokerPartitionAssignment broker_partition_assignments = 1;
} }
////////////////////////////////////////////////// //////////////////////////////////////////////////

825
weed/pb/mq_pb/mq.pb.go
View file

File diff suppressed because it is too large Load diff

28
weed/pb/mq_pb/mq_grpc.pb.go
View file

@ -28,7 +28,7 @@ type SeaweedMessagingClient interface {
CheckSegmentStatus(ctx context.Context, in *CheckSegmentStatusRequest, opts ...grpc.CallOption) (*CheckSegmentStatusResponse, error) CheckSegmentStatus(ctx context.Context, in *CheckSegmentStatusRequest, opts ...grpc.CallOption) (*CheckSegmentStatusResponse, error)
CheckBrokerLoad(ctx context.Context, in *CheckBrokerLoadRequest, opts ...grpc.CallOption) (*CheckBrokerLoadResponse, error) CheckBrokerLoad(ctx context.Context, in *CheckBrokerLoadRequest, opts ...grpc.CallOption) (*CheckBrokerLoadResponse, error)
// control plane for topic partitions // control plane for topic partitions
FindTopicBrokers(ctx context.Context, in *FindTopicBrokersRequest, opts ...grpc.CallOption) (*FindTopicBrokersResponse, error) LookupTopicBrokers(ctx context.Context, in *LookupTopicBrokersRequest, opts ...grpc.CallOption) (*LookupTopicBrokersResponse, error)
// a pub client will call this to get the topic partitions assignment // a pub client will call this to get the topic partitions assignment
RequestTopicPartitions(ctx context.Context, in *RequestTopicPartitionsRequest, opts ...grpc.CallOption) (*RequestTopicPartitionsResponse, error) RequestTopicPartitions(ctx context.Context, in *RequestTopicPartitionsRequest, opts ...grpc.CallOption) (*RequestTopicPartitionsResponse, error)
AssignTopicPartitions(ctx context.Context, in *AssignTopicPartitionsRequest, opts ...grpc.CallOption) (*AssignTopicPartitionsResponse, error) AssignTopicPartitions(ctx context.Context, in *AssignTopicPartitionsRequest, opts ...grpc.CallOption) (*AssignTopicPartitionsResponse, error)
@ -82,9 +82,9 @@ func (c *seaweedMessagingClient) CheckBrokerLoad(ctx context.Context, in *CheckB
return out, nil return out, nil
} }
func (c *seaweedMessagingClient) FindTopicBrokers(ctx context.Context, in *FindTopicBrokersRequest, opts ...grpc.CallOption) (*FindTopicBrokersResponse, error) { func (c *seaweedMessagingClient) LookupTopicBrokers(ctx context.Context, in *LookupTopicBrokersRequest, opts ...grpc.CallOption) (*LookupTopicBrokersResponse, error) {
out := new(FindTopicBrokersResponse) out := new(LookupTopicBrokersResponse)
err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/FindTopicBrokers", in, out, opts...) err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/LookupTopicBrokers", in, out, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -191,7 +191,7 @@ type SeaweedMessagingServer interface {
CheckSegmentStatus(context.Context, *CheckSegmentStatusRequest) (*CheckSegmentStatusResponse, error) CheckSegmentStatus(context.Context, *CheckSegmentStatusRequest) (*CheckSegmentStatusResponse, error)
CheckBrokerLoad(context.Context, *CheckBrokerLoadRequest) (*CheckBrokerLoadResponse, error) CheckBrokerLoad(context.Context, *CheckBrokerLoadRequest) (*CheckBrokerLoadResponse, error)
// control plane for topic partitions // control plane for topic partitions
FindTopicBrokers(context.Context, *FindTopicBrokersRequest) (*FindTopicBrokersResponse, error) LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error)
// a pub client will call this to get the topic partitions assignment // a pub client will call this to get the topic partitions assignment
RequestTopicPartitions(context.Context, *RequestTopicPartitionsRequest) (*RequestTopicPartitionsResponse, error) RequestTopicPartitions(context.Context, *RequestTopicPartitionsRequest) (*RequestTopicPartitionsResponse, error)
AssignTopicPartitions(context.Context, *AssignTopicPartitionsRequest) (*AssignTopicPartitionsResponse, error) AssignTopicPartitions(context.Context, *AssignTopicPartitionsRequest) (*AssignTopicPartitionsResponse, error)
@ -218,8 +218,8 @@ func (UnimplementedSeaweedMessagingServer) CheckSegmentStatus(context.Context, *
func (UnimplementedSeaweedMessagingServer) CheckBrokerLoad(context.Context, *CheckBrokerLoadRequest) (*CheckBrokerLoadResponse, error) { func (UnimplementedSeaweedMessagingServer) CheckBrokerLoad(context.Context, *CheckBrokerLoadRequest) (*CheckBrokerLoadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CheckBrokerLoad not implemented") return nil, status.Errorf(codes.Unimplemented, "method CheckBrokerLoad not implemented")
} }
func (UnimplementedSeaweedMessagingServer) FindTopicBrokers(context.Context, *FindTopicBrokersRequest) (*FindTopicBrokersResponse, error) { func (UnimplementedSeaweedMessagingServer) LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method FindTopicBrokers not implemented") return nil, status.Errorf(codes.Unimplemented, "method LookupTopicBrokers not implemented")
} }
func (UnimplementedSeaweedMessagingServer) RequestTopicPartitions(context.Context, *RequestTopicPartitionsRequest) (*RequestTopicPartitionsResponse, error) { func (UnimplementedSeaweedMessagingServer) RequestTopicPartitions(context.Context, *RequestTopicPartitionsRequest) (*RequestTopicPartitionsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method RequestTopicPartitions not implemented") return nil, status.Errorf(codes.Unimplemented, "method RequestTopicPartitions not implemented")
@ -321,20 +321,20 @@ func _SeaweedMessaging_CheckBrokerLoad_Handler(srv interface{}, ctx context.Cont
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _SeaweedMessaging_FindTopicBrokers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { func _SeaweedMessaging_LookupTopicBrokers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FindTopicBrokersRequest) in := new(LookupTopicBrokersRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }
if interceptor == nil { if interceptor == nil {
return srv.(SeaweedMessagingServer).FindTopicBrokers(ctx, in) return srv.(SeaweedMessagingServer).LookupTopicBrokers(ctx, in)
} }
info := &grpc.UnaryServerInfo{ info := &grpc.UnaryServerInfo{
Server: srv, Server: srv,
FullMethod: "/messaging_pb.SeaweedMessaging/FindTopicBrokers", FullMethod: "/messaging_pb.SeaweedMessaging/LookupTopicBrokers",
} }
handler := func(ctx context.Context, req interface{}) (interface{}, error) { handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedMessagingServer).FindTopicBrokers(ctx, req.(*FindTopicBrokersRequest)) return srv.(SeaweedMessagingServer).LookupTopicBrokers(ctx, req.(*LookupTopicBrokersRequest))
} }
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
@ -464,8 +464,8 @@ var SeaweedMessaging_ServiceDesc = grpc.ServiceDesc{
Handler: _SeaweedMessaging_CheckBrokerLoad_Handler, Handler: _SeaweedMessaging_CheckBrokerLoad_Handler,
}, },
{ {
MethodName: "FindTopicBrokers", MethodName: "LookupTopicBrokers",
Handler: _SeaweedMessaging_FindTopicBrokers_Handler, Handler: _SeaweedMessaging_LookupTopicBrokers_Handler,
}, },
{ {
MethodName: "RequestTopicPartitions", MethodName: "RequestTopicPartitions",

2
weed/pb/remote_pb/remote.pb.go
View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.30.0 // protoc-gen-go v1.28.1
// protoc v4.23.3 // protoc v4.23.3
// source: remote.proto // source: remote.proto

2
weed/pb/s3_pb/s3.pb.go
View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.30.0 // protoc-gen-go v1.28.1
// protoc v4.23.3 // protoc v4.23.3
// source: s3.proto // source: s3.proto

2
weed/pb/volume_server_pb/volume_server.pb.go
View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.30.0 // protoc-gen-go v1.28.1
// protoc v4.23.3 // protoc v4.23.3
// source: volume_server.proto // source: volume_server.proto