seaweedfs/weed/wdclient/masterclient.go

192 lines
5.9 KiB
Go
Raw Normal View History

2018-07-28 09:10:32 +00:00
package wdclient
import (
"context"
2022-01-24 14:09:43 +00:00
"github.com/chrislusf/seaweedfs/weed/stats"
2019-02-18 20:11:52 +00:00
"math/rand"
2018-07-29 04:02:56 +00:00
"time"
2018-07-28 09:10:32 +00:00
2020-11-01 08:40:16 +00:00
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
2018-07-29 04:02:56 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2020-03-04 08:39:47 +00:00
"github.com/chrislusf/seaweedfs/weed/pb"
2018-07-22 00:39:10 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)
2018-07-28 09:10:32 +00:00
type MasterClient struct {
2020-04-13 00:51:31 +00:00
clientType string
clientHost pb.ServerAddress
currentMaster pb.ServerAddress
masters []pb.ServerAddress
2019-02-18 20:11:52 +00:00
grpcDialOption grpc.DialOption
2018-07-29 01:40:31 +00:00
vidMap
2021-11-06 21:23:35 +00:00
OnPeerUpdate func(update *master_pb.ClusterNodeUpdate)
2018-07-28 09:10:32 +00:00
}
func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters []pb.ServerAddress) *MasterClient {
2018-07-28 09:10:32 +00:00
return &MasterClient{
2020-04-13 00:51:31 +00:00
clientType: clientType,
clientHost: clientHost,
2019-02-18 20:11:52 +00:00
masters: masters,
grpcDialOption: grpcDialOption,
2020-11-11 21:13:33 +00:00
vidMap: newVidMap(clientDataCenter),
2018-07-28 09:10:32 +00:00
}
}
func (mc *MasterClient) GetMaster() pb.ServerAddress {
2021-05-21 09:10:44 +00:00
mc.WaitUntilConnected()
2018-07-28 09:10:32 +00:00
return mc.currentMaster
}
func (mc *MasterClient) WaitUntilConnected() {
for mc.currentMaster == "" {
time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond)
}
}
func (mc *MasterClient) KeepConnectedToMaster() {
2020-06-20 19:50:40 +00:00
glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters)
for {
2018-07-28 09:10:32 +00:00
mc.tryAllMasters()
time.Sleep(time.Second)
}
}
func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddress) (leader string) {
for _, master := range mc.masters {
if master == myMasterAddress {
continue
}
if grpcErr := pb.WithMasterClient(false, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
defer cancel()
resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
leader = resp.Leader
return nil
}); grpcErr != nil {
glog.V(0).Infof("connect to %s: %v", master, grpcErr)
}
if leader != "" {
glog.V(0).Infof("existing leader is %s", leader)
return
}
}
glog.V(0).Infof("No existing leader found!")
return
}
2018-07-28 09:10:32 +00:00
func (mc *MasterClient) tryAllMasters() {
var nextHintedLeader pb.ServerAddress
2018-07-28 09:10:32 +00:00
for _, master := range mc.masters {
nextHintedLeader = mc.tryConnectToMaster(master)
for nextHintedLeader != "" {
nextHintedLeader = mc.tryConnectToMaster(nextHintedLeader)
}
mc.currentMaster = ""
2020-11-11 21:13:33 +00:00
mc.vidMap = newVidMap("")
}
}
func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedLeader pb.ServerAddress) {
2021-11-03 06:45:28 +00:00
glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master)
2022-01-24 14:09:43 +00:00
stats.MasterClientConnectCounter.WithLabelValues("total").Inc()
gprcErr := pb.WithMasterClient(true, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := client.KeepConnected(ctx)
if err != nil {
glog.V(1).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err)
stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToKeepConnected).Inc()
return err
}
if err = stream.Send(&master_pb.KeepConnectedRequest{
ClientType: mc.clientType,
ClientAddress: string(mc.clientHost),
Version: util.Version(),
}); err != nil {
2020-06-20 19:50:40 +00:00
glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err)
stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc()
return err
}
2020-06-20 19:50:40 +00:00
glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master)
mc.currentMaster = master
for {
2021-11-06 01:11:40 +00:00
resp, err := stream.Recv()
if err != nil {
2020-06-20 19:50:40 +00:00
glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err)
stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
return err
}
2021-11-06 01:11:40 +00:00
if resp.VolumeLocation != nil {
// maybe the leader is changed
if resp.VolumeLocation.Leader != "" {
glog.V(0).Infof("redirected to leader %v", resp.VolumeLocation.Leader)
nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToleader).Inc()
2021-11-06 01:11:40 +00:00
return nil
}
// process new volume location
loc := Location{
Url: resp.VolumeLocation.Url,
PublicUrl: resp.VolumeLocation.PublicUrl,
DataCenter: resp.VolumeLocation.DataCenter,
GrpcPort: int(resp.VolumeLocation.GrpcPort),
}
for _, newVid := range resp.VolumeLocation.NewVids {
glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid)
mc.addLocation(newVid, loc)
}
for _, deletedVid := range resp.VolumeLocation.DeletedVids {
glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid)
mc.deleteLocation(deletedVid, loc)
}
}
2021-11-06 11:07:38 +00:00
if resp.ClusterNodeUpdate != nil {
update := resp.ClusterNodeUpdate
2021-11-06 21:23:35 +00:00
if mc.OnPeerUpdate != nil {
if update.IsAdd {
glog.V(0).Infof("+ %s %s leader:%v\n", update.NodeType, update.Address, update.IsLeader)
} else {
glog.V(0).Infof("- %s %s leader:%v\n", update.NodeType, update.Address, update.IsLeader)
}
stats.MasterClientConnectCounter.WithLabelValues(stats.OnPeerUpdate).Inc()
2021-11-06 21:23:35 +00:00
mc.OnPeerUpdate(update)
2021-11-06 11:07:38 +00:00
}
}
}
})
if gprcErr != nil {
stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc()
glog.V(1).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr)
}
return
}
func (mc *MasterClient) WithClient(streamingMode bool, fn func(client master_pb.SeaweedClient) error) error {
2020-11-01 10:36:43 +00:00
return util.Retry("master grpc", func() error {
2020-11-01 08:40:16 +00:00
for mc.currentMaster == "" {
time.Sleep(3 * time.Second)
}
return pb.WithMasterClient(streamingMode, mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
2020-11-01 08:40:16 +00:00
return fn(client)
})
2019-03-19 12:19:37 +00:00
})
}