seaweedfs/weed/pb/grpc_client_server.go

273 lines
7.6 KiB
Go
Raw Normal View History

2020-03-04 08:39:47 +00:00
package pb
2018-07-04 02:07:55 +00:00
import (
2019-03-16 00:20:24 +00:00
"context"
2018-12-07 09:25:01 +00:00
"fmt"
2021-03-03 04:59:39 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2021-09-07 23:43:54 +00:00
"github.com/chrislusf/seaweedfs/weed/util"
"math/rand"
"net/http"
"strconv"
"strings"
2018-12-07 09:25:01 +00:00
"sync"
2018-07-04 02:07:55 +00:00
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/keepalive"
2020-03-04 08:39:47 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
2018-07-04 02:07:55 +00:00
)
const (
Max_Message_Size = 1 << 30 // 1 GB
)
2018-12-07 09:25:01 +00:00
var (
// cache grpc connections
grpcClients = make(map[string]*versionedGrpcClient)
2018-12-07 09:25:01 +00:00
grpcClientsLock sync.Mutex
)
type versionedGrpcClient struct {
*grpc.ClientConn
version int
errCount int
}
2019-04-11 16:27:05 +00:00
func init() {
2019-04-16 05:00:50 +00:00
http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
}
2019-02-16 20:49:58 +00:00
func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
var options []grpc.ServerOption
options = append(options,
grpc.KeepaliveParams(keepalive.ServerParameters{
Time: 10 * time.Second, // wait time before ping if no activity
Timeout: 20 * time.Second, // ping timeout
MaxConnectionAge: 10 * time.Hour,
}),
grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
MinTime: 60 * time.Second, // min time a client should wait before sending a ping
PermitWithoutStream: true,
}),
grpc.MaxRecvMsgSize(Max_Message_Size),
grpc.MaxSendMsgSize(Max_Message_Size),
)
2019-02-16 20:49:58 +00:00
for _, opt := range opts {
if opt != nil {
options = append(options, opt)
}
}
return grpc.NewServer(options...)
2018-07-04 02:07:55 +00:00
}
2019-03-16 00:20:24 +00:00
func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
// opts = append(opts, grpc.WithBlock())
// opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
2019-02-16 20:49:58 +00:00
var options []grpc.DialOption
options = append(options,
2019-02-18 20:11:52 +00:00
// grpc.WithInsecure(),
grpc.WithDefaultCallOptions(
grpc.MaxCallSendMsgSize(Max_Message_Size),
grpc.MaxCallRecvMsgSize(Max_Message_Size),
),
2019-02-16 20:49:58 +00:00
grpc.WithKeepaliveParams(keepalive.ClientParameters{
2020-02-27 00:52:57 +00:00
Time: 30 * time.Second, // client ping server if no activity for this long
Timeout: 20 * time.Second,
PermitWithoutStream: true,
2019-02-16 20:49:58 +00:00
}))
for _, opt := range opts {
if opt != nil {
options = append(options, opt)
}
}
2019-03-16 00:20:24 +00:00
return grpc.DialContext(ctx, address, options...)
2018-07-04 02:07:55 +00:00
}
2018-12-07 09:25:01 +00:00
func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
2018-12-07 09:25:01 +00:00
grpcClientsLock.Lock()
2020-07-04 05:25:35 +00:00
defer grpcClientsLock.Unlock()
2018-12-07 09:25:01 +00:00
existingConnection, found := grpcClients[address]
if found {
2020-07-04 05:25:35 +00:00
return existingConnection, nil
2018-12-07 09:25:01 +00:00
}
grpcConnection, err := GrpcDial(context.Background(), address, opts...)
2018-12-07 09:25:01 +00:00
if err != nil {
2020-07-04 05:25:35 +00:00
return nil, fmt.Errorf("fail to dial %s: %v", address, err)
2018-12-07 09:25:01 +00:00
}
vgc := &versionedGrpcClient{
grpcConnection,
rand.Int(),
0,
}
grpcClients[address] = vgc
2018-12-07 09:25:01 +00:00
return vgc, nil
2020-07-04 05:25:35 +00:00
}
func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {
vgc, err := getOrCreateConnection(address, opts...)
2018-12-07 09:25:01 +00:00
if err != nil {
2020-07-04 05:25:35 +00:00
return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
2018-12-07 09:25:01 +00:00
}
executionErr := fn(vgc.ClientConn)
if executionErr != nil {
2021-08-14 12:10:30 +00:00
if strings.Contains(executionErr.Error(), "transport") ||
strings.Contains(executionErr.Error(), "connection closed") {
grpcClientsLock.Lock()
if t, ok := grpcClients[address]; ok {
if t.version == vgc.version {
vgc.Close()
delete(grpcClients, address)
}
}
grpcClientsLock.Unlock()
}
}
return executionErr
2018-12-07 09:25:01 +00:00
}
func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
2021-03-06 22:25:44 +00:00
return ParseServerAddress(server, 10000)
}
func ParseServersToGrpcAddresses(servers []string) (serverGrpcAddresses []string, err error) {
for _, server := range servers {
if serverGrpcAddress, parseErr := ParseServerToGrpcAddress(server); parseErr == nil {
serverGrpcAddresses = append(serverGrpcAddresses, serverGrpcAddress)
} else {
return nil, parseErr
}
}
return
}
2021-03-06 22:25:44 +00:00
func ParseServerAddress(server string, deltaPort int) (newServerAddress string, err error) {
2021-03-03 04:59:39 +00:00
host, port, parseErr := hostAndPort(server)
if parseErr != nil {
return "", fmt.Errorf("server port parse error: %v", parseErr)
}
2021-03-06 22:25:44 +00:00
newPort := int(port) + deltaPort
2021-09-07 23:43:54 +00:00
return util.JoinHostPort(host, newPort), nil
}
2021-03-03 04:59:39 +00:00
func hostAndPort(address string) (host string, port uint64, err error) {
colonIndex := strings.LastIndex(address, ":")
if colonIndex < 0 {
return "", 0, fmt.Errorf("server should have hostname:port format: %v", address)
}
port, err = strconv.ParseUint(address[colonIndex+1:], 10, 64)
if err != nil {
2021-03-03 04:59:56 +00:00
return "", 0, fmt.Errorf("server port parse error: %v", err)
}
2021-03-03 04:59:39 +00:00
return address[:colonIndex], port, err
}
func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
host, port, parseErr := hostAndPort(server)
if parseErr != nil {
2021-03-03 04:59:39 +00:00
glog.Fatalf("server address %s parse error: %v", server, parseErr)
}
grpcPort := int(port) + 10000
2021-09-07 23:43:54 +00:00
return util.JoinHostPort(host, grpcPort)
}
2020-03-04 08:39:47 +00:00
func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) {
2021-03-03 04:59:39 +00:00
host, grpcPort, parseErr := hostAndPort(grpcAddress)
if parseErr != nil {
2021-03-03 04:59:39 +00:00
glog.Fatalf("server grpc address %s parse error: %v", grpcAddress, parseErr)
}
port := int(grpcPort) - 10000
2021-09-07 23:43:54 +00:00
return util.JoinHostPort(host, port)
}
2020-03-04 08:39:47 +00:00
func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {
masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
if parseErr != nil {
return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
}
return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := master_pb.NewSeaweedClient(grpcConnection)
return fn(client)
}, masterGrpcAddress, grpcDialOption)
}
func WithOneOfGrpcMasterClients(masterGrpcAddresses []string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
for _, masterGrpcAddress := range masterGrpcAddresses {
err = WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := master_pb.NewSeaweedClient(grpcConnection)
return fn(client)
}, masterGrpcAddress, grpcDialOption)
if err == nil {
return nil
}
}
return err
}
func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {
return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
return fn(client)
}, brokerGrpcAddress, grpcDialOption)
}
2020-03-04 08:39:47 +00:00
func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
if parseErr != nil {
return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
}
return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)
}
func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, filerGrpcAddress, grpcDialOption)
}
func WithOneOfGrpcFilerClients(filerGrpcAddresses []string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) {
for _, filerGrpcAddress := range filerGrpcAddresses {
err = WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, filerGrpcAddress, grpcDialOption)
if err == nil {
return nil
}
}
return err
}