seaweedfs/weed/cluster/lock_client.go

211 lines
5.3 KiB
Go
Raw Normal View History

2023-06-26 00:38:34 +00:00
package cluster
import (
"context"
"fmt"
2023-06-26 03:30:20 +00:00
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
2023-06-26 00:38:34 +00:00
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"time"
)
type LockClient struct {
grpcDialOption grpc.DialOption
maxLockDuration time.Duration
sleepDuration time.Duration
2023-09-16 22:05:38 +00:00
seedFiler pb.ServerAddress
2023-06-26 00:38:34 +00:00
}
2023-09-16 22:05:38 +00:00
func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
2023-06-26 00:38:34 +00:00
return &LockClient{
grpcDialOption: grpcDialOption,
maxLockDuration: 5 * time.Second,
2023-09-16 22:05:38 +00:00
sleepDuration: 2473 * time.Millisecond,
seedFiler: seedFiler,
2023-06-26 00:38:34 +00:00
}
}
type LiveLock struct {
key string
renewToken string
expireAtNs int64
filer pb.ServerAddress
cancelCh chan struct{}
grpcDialOption grpc.DialOption
isLocked bool
2023-09-16 22:05:38 +00:00
owner string
lc *LockClient
2023-06-26 00:38:34 +00:00
}
2023-09-16 22:05:38 +00:00
// NewLock creates a lock with a very long duration
func (lc *LockClient) NewLock(key string, owner string) (lock *LiveLock) {
return lc.doNewLock(key, lock_manager.MaxDuration, owner)
2023-06-26 02:31:25 +00:00
}
2023-09-16 22:05:38 +00:00
// StartLock starts a goroutine to lock the key and returns immediately.
func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
lock = &LiveLock{
key: key,
filer: lc.seedFiler,
cancelCh: make(chan struct{}),
expireAtNs: time.Now().Add(lock_manager.MaxDuration).UnixNano(),
grpcDialOption: lc.grpcDialOption,
owner: owner,
lc: lc,
}
go func() {
2023-10-01 18:33:56 +00:00
util.RetryUntil("create lock:"+key, func() error {
2023-09-16 22:05:38 +00:00
errorMessage, err := lock.doLock(lock_manager.MaxDuration)
if err != nil {
glog.Infof("create lock %s: %s", key, err)
time.Sleep(time.Second)
return err
}
if errorMessage != "" {
glog.Infof("create lock %s: %s", key, errorMessage)
time.Sleep(time.Second)
return fmt.Errorf("%v", errorMessage)
}
lock.isLocked = true
return nil
}, func(err error) (shouldContinue bool) {
if err != nil {
glog.Warningf("create lock %s: %s", key, err)
time.Sleep(time.Second)
}
return lock.renewToken == ""
})
lc.keepLock(lock)
}()
return
2023-06-26 02:31:25 +00:00
}
2023-09-16 22:05:38 +00:00
func (lc *LockClient) doNewLock(key string, lockDuration time.Duration, owner string) (lock *LiveLock) {
2023-06-26 00:38:34 +00:00
lock = &LiveLock{
key: key,
2023-09-16 22:05:38 +00:00
filer: lc.seedFiler,
2023-06-26 00:38:34 +00:00
cancelCh: make(chan struct{}),
expireAtNs: time.Now().Add(lockDuration).UnixNano(),
grpcDialOption: lc.grpcDialOption,
2023-09-16 22:05:38 +00:00
owner: owner,
lc: lc,
2023-06-26 00:38:34 +00:00
}
var needRenewal bool
if lockDuration > lc.maxLockDuration {
lockDuration = lc.maxLockDuration
needRenewal = true
}
2023-10-01 18:33:56 +00:00
util.RetryUntil("create lock:"+key, func() error {
2023-06-26 00:38:34 +00:00
errorMessage, err := lock.doLock(lockDuration)
if err != nil {
2023-09-16 22:05:38 +00:00
time.Sleep(time.Second)
2023-06-26 00:38:34 +00:00
return err
}
if errorMessage != "" {
2023-09-16 22:05:38 +00:00
time.Sleep(time.Second)
2023-06-26 00:38:34 +00:00
return fmt.Errorf("%v", errorMessage)
}
2023-09-16 22:05:38 +00:00
lock.isLocked = true
2023-06-26 00:38:34 +00:00
return nil
}, func(err error) (shouldContinue bool) {
if err != nil {
glog.Warningf("create lock %s: %s", key, err)
}
return lock.renewToken == ""
})
if needRenewal {
go lc.keepLock(lock)
}
return
}
func (lock *LiveLock) IsLocked() bool {
return lock.isLocked
}
2023-09-16 22:05:38 +00:00
func (lock *LiveLock) StopLock() error {
2023-06-26 00:38:34 +00:00
close(lock.cancelCh)
2023-09-16 22:05:38 +00:00
if !lock.isLocked {
return nil
}
2023-06-26 00:38:34 +00:00
return pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
2023-09-16 22:05:38 +00:00
_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
2023-06-26 00:38:34 +00:00
Name: lock.key,
RenewToken: lock.renewToken,
})
return err
})
}
func (lc *LockClient) keepLock(lock *LiveLock) {
2023-09-16 22:05:38 +00:00
ticker := time.Tick(lc.sleepDuration)
2023-06-26 00:38:34 +00:00
for {
select {
2023-09-16 22:05:38 +00:00
case <-ticker:
2023-06-26 00:38:34 +00:00
// renew the lock if lock.expireAtNs is still greater than now
2023-10-01 18:33:56 +00:00
util.RetryUntil("keep lock:"+lock.key, func() error {
2023-06-26 00:38:34 +00:00
lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
if lockDuration > lc.maxLockDuration {
lockDuration = lc.maxLockDuration
}
if lockDuration <= 0 {
return nil
}
errorMessage, err := lock.doLock(lockDuration)
if err != nil {
lock.isLocked = false
2023-09-16 22:05:38 +00:00
time.Sleep(time.Second)
2023-06-26 00:38:34 +00:00
return err
}
if errorMessage != "" {
lock.isLocked = false
2023-09-16 22:05:38 +00:00
time.Sleep(time.Second)
2023-06-26 00:38:34 +00:00
return fmt.Errorf("%v", errorMessage)
}
return nil
}, func(err error) (shouldContinue bool) {
if err == nil {
return false
}
glog.Warningf("keep lock %s: %v", lock.key, err)
return true
})
if !lock.isLocked {
return
}
case <-lock.cancelCh:
return
}
}
}
func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
err = pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
2023-09-16 22:05:38 +00:00
resp, err := client.DistributedLock(context.Background(), &filer_pb.LockRequest{
2023-06-26 00:38:34 +00:00
Name: lock.key,
SecondsToLock: int64(lockDuration.Seconds()),
RenewToken: lock.renewToken,
IsMoved: false,
2023-09-16 22:05:38 +00:00
Owner: lock.owner,
2023-06-26 00:38:34 +00:00
})
if err == nil {
lock.renewToken = resp.RenewToken
}
if resp != nil {
errorMessage = resp.Error
if resp.MovedTo != "" {
lock.filer = pb.ServerAddress(resp.MovedTo)
2023-09-16 22:05:38 +00:00
lock.lc.seedFiler = lock.filer
2023-06-26 00:38:34 +00:00
}
}
return err
})
return
}