seaweedfs/weed/server/volume_grpc_client_to_master.go

241 lines
7.9 KiB
Go
Raw Normal View History

2017-01-10 09:01:12 +00:00
package weed_server
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/operation"
2019-04-15 06:00:37 +00:00
"time"
"google.golang.org/grpc"
2020-03-04 08:39:47 +00:00
"github.com/chrislusf/seaweedfs/weed/pb"
2019-02-18 20:11:52 +00:00
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/backend"
"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
"golang.org/x/net/context"
2017-01-10 09:01:12 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2018-05-10 06:11:54 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
2018-07-04 02:07:55 +00:00
"github.com/chrislusf/seaweedfs/weed/util"
2017-01-10 09:01:12 +00:00
)
func (vs *VolumeServer) GetMaster() string {
return vs.currentMaster
}
func (vs *VolumeServer) checkWithMaster() (err error) {
isConnected := false
for !isConnected {
for _, master := range vs.SeedMasterNodes {
err = operation.WithMasterServerClient(master, vs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
resp, err := masterClient.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return fmt.Errorf("get master %s configuration: %v", master, err)
}
vs.metricsAddress, vs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
backend.LoadFromPbStorageBackends(resp.StorageBackends)
return nil
})
if err == nil {
return
} else {
glog.V(0).Infof("checkWithMaster %s: %v", master, err)
}
}
time.Sleep(1790 * time.Millisecond)
}
return
}
2017-01-10 09:01:12 +00:00
func (vs *VolumeServer) heartbeat() {
glog.V(0).Infof("Volume server start with seed master nodes: %v", vs.SeedMasterNodes)
2017-01-10 09:01:12 +00:00
vs.store.SetDataCenter(vs.dataCenter)
vs.store.SetRack(vs.rack)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.volume")
2019-02-18 20:11:52 +00:00
var err error
var newLeader string
2020-09-14 04:25:51 +00:00
for vs.isHeartbeating {
for _, master := range vs.SeedMasterNodes {
if newLeader != "" {
// the new leader may actually is the same master
// need to wait a bit before adding itself
time.Sleep(3 * time.Second)
master = newLeader
}
2020-03-04 08:39:47 +00:00
masterGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(master)
if parseErr != nil {
2019-04-16 08:15:30 +00:00
glog.V(0).Infof("failed to parse master grpc %v: %v", masterGrpcAddress, parseErr)
continue
}
2019-05-28 05:54:58 +00:00
vs.store.MasterAddress = master
2020-02-26 06:23:59 +00:00
newLeader, err = vs.doHeartbeat(master, masterGrpcAddress, grpcDialOption, time.Duration(vs.pulseSeconds)*time.Second)
if err != nil {
glog.V(0).Infof("heartbeat error: %v", err)
time.Sleep(time.Duration(vs.pulseSeconds) * time.Second)
2019-04-15 16:09:46 +00:00
newLeader = ""
2019-05-28 05:54:58 +00:00
vs.store.MasterAddress = ""
}
2020-09-14 04:25:51 +00:00
if !vs.isHeartbeating {
break
}
2017-01-10 09:01:12 +00:00
}
}
}
2020-09-14 04:25:51 +00:00
func (vs *VolumeServer) StopHeartbeat() (isAlreadyStopping bool) {
if !vs.isHeartbeating {
return true
}
vs.isHeartbeating = false
close(vs.stopChan)
2020-09-14 04:25:51 +00:00
return false
}
2020-02-26 06:23:59 +00:00
func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDialOption grpc.DialOption, sleepInterval time.Duration) (newLeader string, err error) {
2017-01-10 09:01:12 +00:00
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
grpcConection, err := pb.GrpcDial(ctx, masterGrpcAddress, grpcDialOption)
2017-01-10 09:01:12 +00:00
if err != nil {
2019-01-11 10:23:31 +00:00
return "", fmt.Errorf("fail to dial %s : %v", masterNode, err)
2017-01-10 09:01:12 +00:00
}
defer grpcConection.Close()
2018-05-10 06:11:54 +00:00
client := master_pb.NewSeaweedClient(grpcConection)
stream, err := client.SendHeartbeat(ctx)
2017-01-10 09:01:12 +00:00
if err != nil {
glog.V(0).Infof("SendHeartbeat to %s: %v", masterNode, err)
return "", err
2017-01-10 09:01:12 +00:00
}
glog.V(0).Infof("Heartbeat to: %v", masterNode)
vs.currentMaster = masterNode
2017-01-10 09:01:12 +00:00
doneChan := make(chan error, 1)
2017-01-10 09:01:12 +00:00
go func() {
for {
in, err := stream.Recv()
if err != nil {
doneChan <- err
2017-01-10 09:01:12 +00:00
return
}
if in.GetVolumeSizeLimit() != 0 && vs.store.GetVolumeSizeLimit() != in.GetVolumeSizeLimit() {
2019-04-15 06:00:37 +00:00
vs.store.SetVolumeSizeLimit(in.GetVolumeSizeLimit())
if vs.store.MaybeAdjustVolumeMax() {
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", vs.currentMaster, err)
}
}
}
if in.GetLeader() != "" && vs.currentMaster != in.GetLeader() {
glog.V(0).Infof("Volume Server found a new master newLeader: %v instead of %v", in.GetLeader(), vs.currentMaster)
newLeader = in.GetLeader()
doneChan <- nil
return
}
2017-01-10 09:01:12 +00:00
}
}()
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
volumeTickChan := time.Tick(sleepInterval)
ecShardTickChan := time.Tick(17 * sleepInterval)
2017-01-10 09:01:12 +00:00
for {
select {
2019-04-20 18:35:20 +00:00
case volumeMessage := <-vs.store.NewVolumesChan:
deltaBeat := &master_pb.Heartbeat{
2019-04-30 03:22:19 +00:00
NewVolumes: []*master_pb.VolumeShortInformationMessage{
2019-04-20 18:35:20 +00:00
&volumeMessage,
},
}
2019-04-20 18:35:20 +00:00
glog.V(1).Infof("volume server %s:%d adds volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id)
if err = stream.Send(deltaBeat); err != nil {
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
case ecShardMessage := <-vs.store.NewEcShardsChan:
deltaBeat := &master_pb.Heartbeat{
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
&ecShardMessage,
},
}
glog.V(1).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id,
erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds())
if err = stream.Send(deltaBeat); err != nil {
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
2019-04-20 18:35:20 +00:00
case volumeMessage := <-vs.store.DeletedVolumesChan:
deltaBeat := &master_pb.Heartbeat{
2019-04-30 03:22:19 +00:00
DeletedVolumes: []*master_pb.VolumeShortInformationMessage{
2019-04-20 18:35:20 +00:00
&volumeMessage,
},
}
2019-04-20 18:35:20 +00:00
glog.V(1).Infof("volume server %s:%d deletes volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id)
if err = stream.Send(deltaBeat); err != nil {
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
case ecShardMessage := <-vs.store.DeletedEcShardsChan:
deltaBeat := &master_pb.Heartbeat{
DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{
&ecShardMessage,
},
}
glog.V(1).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id,
erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds())
if err = stream.Send(deltaBeat); err != nil {
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
case <-volumeTickChan:
2020-09-14 04:25:51 +00:00
glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port)
vs.store.MaybeAdjustVolumeMax()
2020-09-14 04:25:51 +00:00
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
case <-ecShardTickChan:
2020-08-31 03:12:04 +00:00
glog.V(4).Infof("volume server %s:%d ec heartbeat", vs.store.Ip, vs.store.Port)
if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
case err = <-doneChan:
return
2020-09-14 04:25:51 +00:00
case <-vs.stopChan:
var volumeMessages []*master_pb.VolumeInformationMessage
emptyBeat := &master_pb.Heartbeat{
Ip: vs.store.Ip,
Port: uint32(vs.store.Port),
PublicUrl: vs.store.PublicUrl,
MaxVolumeCount: uint32(0),
MaxFileKey: uint64(0),
DataCenter: vs.store.GetDataCenter(),
Rack: vs.store.GetRack(),
Volumes: volumeMessages,
HasNoVolumes: len(volumeMessages) == 0,
}
2020-10-30 22:21:54 +00:00
glog.V(1).Infof("volume server %s:%d stops and deletes all volumes", vs.store.Ip, vs.store.Port)
if err = stream.Send(emptyBeat); err != nil {
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
2020-09-14 04:25:51 +00:00
return
2017-01-10 09:01:12 +00:00
}
}
}