seaweedfs/weed/filer/filer.go

377 lines
11 KiB
Go
Raw Normal View History

2020-09-01 07:21:19 +00:00
package filer
2018-05-11 09:20:15 +00:00
import (
2018-07-29 04:02:56 +00:00
"context"
2018-05-11 09:20:15 +00:00
"fmt"
2023-06-25 07:58:21 +00:00
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
2018-05-27 18:52:26 +00:00
"os"
"sort"
2018-05-27 18:52:26 +00:00
"strings"
2018-05-12 20:45:29 +00:00
"time"
2018-07-29 04:02:56 +00:00
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
2018-05-11 09:20:15 +00:00
)
const (
LogFlushInterval = time.Minute
PaginationSize = 1024
2020-09-05 21:08:59 +00:00
FilerStoreId = "filer.store.id"
)
2018-12-03 03:42:50 +00:00
var (
2020-09-02 04:59:26 +00:00
OS_UID = uint32(os.Getuid())
OS_GID = uint32(os.Getgid())
2018-12-03 03:42:50 +00:00
)
2018-05-11 09:20:15 +00:00
type Filer struct {
2022-09-01 16:02:44 +00:00
UniqueFilerId int32
UniqueFilerEpoch int32
2020-09-24 10:06:44 +00:00
Store VirtualFilerStore
MasterClient *wdclient.MasterClient
fileIdDeletionQueue *util.UnboundedQueue
GrpcDialOption grpc.DialOption
DirBucketsPath string
Cipher bool
2020-07-05 22:52:36 +00:00
LocalMetaLogBuffer *log_buffer.LogBuffer
2020-04-12 21:03:07 +00:00
metaLogCollection string
metaLogReplication string
2020-07-13 00:31:24 +00:00
MetaAggregator *MetaAggregator
Signature int32
2020-11-15 22:06:03 +00:00
FilerConf *FilerConf
RemoteStorage *FilerRemoteStorage
2023-06-25 21:30:58 +00:00
Dlm *lock_manager.DistributedLockManager
2018-05-11 09:20:15 +00:00
}
2022-05-02 04:59:16 +00:00
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
2018-11-21 04:56:28 +00:00
f := &Filer{
2022-07-03 07:29:25 +00:00
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
fileIdDeletionQueue: util.NewUnboundedQueue(),
GrpcDialOption: grpcDialOption,
2020-11-15 22:06:03 +00:00
FilerConf: NewFilerConf(),
RemoteStorage: NewFilerRemoteStorage(),
UniqueFilerId: util.RandomInt32(),
2023-06-26 03:30:20 +00:00
Dlm: lock_manager.NewDistributedLockManager(filerHost),
2018-05-11 09:20:15 +00:00
}
if f.UniqueFilerId < 0 {
f.UniqueFilerId = -f.UniqueFilerId
}
2021-06-27 12:51:28 +00:00
f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn)
2020-04-12 21:03:07 +00:00
f.metaLogCollection = collection
f.metaLogReplication = replication
2020-11-11 23:10:06 +00:00
2018-11-21 04:56:28 +00:00
go f.loopProcessingDeletion()
return f
2018-05-11 09:20:15 +00:00
}
func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) {
if len(existingNodes) == 0 {
return
}
sort.Slice(existingNodes, func(i, j int) bool {
return existingNodes[i].CreatedAtNs < existingNodes[j].CreatedAtNs
})
earliestNode := existingNodes[0]
if earliestNode.Address == string(self) {
return
}
glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId)
f.UniqueFilerEpoch++
2023-03-22 06:01:49 +00:00
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "bootstrap",
ClientId: f.UniqueFilerId,
ClientEpoch: f.UniqueFilerEpoch,
SelfSignature: f.Signature,
PathPrefix: "/",
AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil,
StartTsNs: snapshotTime.UnixNano(),
StopTsNs: 0,
EventErrorType: pb.FatalOnError,
}
err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
return Replay(f.Store, resp)
})
return
}
func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, startFrom time.Time) {
2020-07-13 00:31:24 +00:00
2023-06-25 22:28:16 +00:00
var snapshot []pb.ServerAddress
for _, node := range existingNodes {
address := pb.ServerAddress(node.Address)
snapshot = append(snapshot, address)
}
f.Dlm.LockRing.SetSnapshot(snapshot)
glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
2021-11-06 21:23:35 +00:00
f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
2023-06-25 07:58:21 +00:00
f.MasterClient.SetOnPeerUpdateFn(func(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
if update.NodeType != cluster.FilerType {
return
}
address := pb.ServerAddress(update.Address)
if update.IsAdd {
2023-06-25 21:30:58 +00:00
f.Dlm.LockRing.AddServer(address)
2023-06-25 07:58:21 +00:00
} else {
2023-06-25 21:30:58 +00:00
f.Dlm.LockRing.RemoveServer(address)
2023-06-25 07:58:21 +00:00
}
f.MetaAggregator.OnPeerUpdate(update, startFrom)
})
2021-11-06 21:23:35 +00:00
2022-05-30 23:28:36 +00:00
for _, peerUpdate := range existingNodes {
f.MetaAggregator.OnPeerUpdate(peerUpdate, startFrom)
2020-07-13 00:31:24 +00:00
}
2020-09-06 07:11:46 +00:00
2021-11-06 21:23:35 +00:00
}
2020-07-13 00:31:24 +00:00
func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) {
2022-07-12 09:00:54 +00:00
return cluster.ListExistingPeerUpdates(f.GetMaster(), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
2020-07-13 00:31:24 +00:00
}
func (f *Filer) SetStore(store FilerStore) (isFresh bool) {
2020-07-13 15:19:48 +00:00
f.Store = NewFilerStoreWrapper(store)
2020-09-05 21:08:59 +00:00
return f.setOrLoadFilerStoreSignature(store)
2020-09-05 21:08:59 +00:00
}
func (f *Filer) setOrLoadFilerStoreSignature(store FilerStore) (isFresh bool) {
2020-09-05 21:08:59 +00:00
storeIdBytes, err := store.KvGet(context.Background(), []byte(FilerStoreId))
if err == ErrKvNotFound || err == nil && len(storeIdBytes) == 0 {
f.Signature = util.RandomInt32()
storeIdBytes = make([]byte, 4)
util.Uint32toBytes(storeIdBytes, uint32(f.Signature))
if err = store.KvPut(context.Background(), []byte(FilerStoreId), storeIdBytes); err != nil {
glog.Fatalf("set %s=%d : %v", FilerStoreId, f.Signature, err)
}
glog.V(0).Infof("create %s to %d", FilerStoreId, f.Signature)
return true
2020-09-05 21:08:59 +00:00
} else if err == nil && len(storeIdBytes) == 4 {
f.Signature = int32(util.BytesToUint32(storeIdBytes))
glog.V(0).Infof("existing %s = %d", FilerStoreId, f.Signature)
} else {
glog.Fatalf("read %v=%v : %v", FilerStoreId, string(storeIdBytes), err)
}
return false
2018-05-11 09:20:15 +00:00
}
func (f *Filer) GetStore() (store FilerStore) {
2020-07-13 15:19:48 +00:00
return f.Store
}
func (fs *Filer) GetMaster() pb.ServerAddress {
return fs.MasterClient.GetMaster()
2018-07-28 09:10:32 +00:00
}
2021-11-06 21:23:35 +00:00
func (fs *Filer) KeepMasterClientConnected() {
fs.MasterClient.KeepConnectedToMaster()
2018-07-28 09:10:32 +00:00
}
func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
2020-07-13 15:19:48 +00:00
return f.Store.BeginTransaction(ctx)
}
func (f *Filer) CommitTransaction(ctx context.Context) error {
2020-07-13 15:19:48 +00:00
return f.Store.CommitTransaction(ctx)
}
func (f *Filer) RollbackTransaction(ctx context.Context) error {
2020-07-13 15:19:48 +00:00
return f.Store.RollbackTransaction(ctx)
}
func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error {
2018-05-12 20:45:29 +00:00
2018-12-03 07:16:17 +00:00
if string(entry.FullPath) == "/" {
return nil
}
oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
2018-05-12 20:45:29 +00:00
/*
2018-05-27 18:52:26 +00:00
if !hasWritePermission(lastDirectoryEntry, entry) {
glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
}
*/
2018-05-12 20:45:29 +00:00
if oldEntry == nil {
if !skipCreateParentDir {
dirParts := strings.Split(string(entry.FullPath), "/")
if err := f.ensureParentDirectoryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
return err
}
}
2020-10-24 16:42:54 +00:00
glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())
2020-07-13 15:19:48 +00:00
if err := f.Store.InsertEntry(ctx, entry); err != nil {
2019-06-21 06:45:30 +00:00
glog.Errorf("insert entry %s: %v", entry.FullPath, err)
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
}
} else {
if o_excl {
2020-01-25 06:01:51 +00:00
glog.V(3).Infof("EEXIST: entry %s already exists", entry.FullPath)
return fmt.Errorf("EEXIST: entry %s already exists", entry.FullPath)
}
2020-10-25 03:11:31 +00:00
glog.V(4).Infof("UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
2019-03-15 22:55:34 +00:00
if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
2019-06-21 06:45:30 +00:00
glog.Errorf("update entry %s: %v", entry.FullPath, err)
return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
}
2018-05-12 20:45:29 +00:00
}
f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
2018-08-13 08:20:49 +00:00
f.deleteChunksIfNotNew(oldEntry, entry)
2020-01-25 06:13:06 +00:00
glog.V(4).Infof("CreateEntry %s: created", entry.FullPath)
2018-05-12 20:45:29 +00:00
return nil
2018-05-11 09:20:15 +00:00
}
func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, dirParts []string, level int, isFromOtherCluster bool) (err error) {
if level == 0 {
return nil
}
dirPath := "/" + util.Join(dirParts[:level]...)
// fmt.Printf("%d directory: %+v\n", i, dirPath)
// check the store directly
glog.V(4).Infof("find uncached directory: %s", dirPath)
dirEntry, _ := f.FindEntry(ctx, util.FullPath(dirPath))
// no such existing directory
if dirEntry == nil {
// ensure parent directory
if err = f.ensureParentDirectoryEntry(ctx, entry, dirParts, level-1, isFromOtherCluster); err != nil {
return err
}
// create the directory
now := time.Now()
dirEntry = &Entry{
FullPath: util.FullPath(dirPath),
Attr: Attr{
Mtime: now,
Crtime: now,
Mode: os.ModeDir | entry.Mode | 0111,
Uid: entry.Uid,
Gid: entry.Gid,
UserName: entry.UserName,
GroupNames: entry.GroupNames,
},
}
glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
mkdirErr := f.Store.InsertEntry(ctx, dirEntry)
if mkdirErr != nil {
if _, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound {
glog.V(3).Infof("mkdir %s: %v", dirPath, mkdirErr)
return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
}
} else {
if !strings.HasPrefix("/"+util.Join(dirParts[:]...), SystemLogDir) {
f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
}
}
} else if !dirEntry.IsDirectory() {
glog.Errorf("CreateEntry %s: %s should be a directory", entry.FullPath, dirPath)
return fmt.Errorf("%s is a file", dirPath)
}
return nil
}
2019-03-15 22:55:34 +00:00
func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
if oldEntry != nil {
2020-11-30 11:11:52 +00:00
entry.Attr.Crtime = oldEntry.Attr.Crtime
if oldEntry.IsDirectory() && !entry.IsDirectory() {
2021-03-31 03:36:06 +00:00
glog.Errorf("existing %s is a directory", oldEntry.FullPath)
return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
}
if !oldEntry.IsDirectory() && entry.IsDirectory() {
2021-03-31 03:36:06 +00:00
glog.Errorf("existing %s is a file", oldEntry.FullPath)
return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
}
}
2020-07-13 15:19:48 +00:00
return f.Store.UpdateEntry(ctx, entry)
2018-05-11 09:20:15 +00:00
}
2020-12-11 06:23:22 +00:00
var (
Root = &Entry{
FullPath: "/",
Attr: Attr{
Mtime: time.Now(),
Crtime: time.Now(),
Mode: os.ModeDir | 0755,
Uid: OS_UID,
Gid: OS_GID,
},
}
)
2018-12-03 03:42:50 +00:00
2020-12-11 06:23:22 +00:00
func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, err error) {
2018-12-03 03:42:50 +00:00
if string(p) == "/" {
2020-12-11 06:23:22 +00:00
return Root, nil
2018-12-03 03:42:50 +00:00
}
2020-07-13 15:19:48 +00:00
entry, err = f.Store.FindEntry(ctx, p)
if entry != nil && entry.TtlSec > 0 {
if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
f.Store.DeleteOneEntry(ctx, entry)
return nil, filer_pb.ErrNotFound
}
}
return
2018-05-11 09:20:15 +00:00
}
2021-01-16 07:56:24 +00:00
func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) {
lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
select {
case <-ctx.Done():
return false
default:
if entry.TtlSec > 0 {
if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
f.Store.DeleteOneEntry(ctx, entry)
expiredCount++
return true
}
}
return eachEntryFunc(entry)
}
2021-01-16 07:56:24 +00:00
})
if err != nil {
return expiredCount, lastFileName, err
}
return
2018-05-11 09:20:15 +00:00
}
2020-03-15 03:30:26 +00:00
func (f *Filer) Shutdown() {
2020-07-05 22:52:36 +00:00
f.LocalMetaLogBuffer.Shutdown()
2020-07-13 15:19:48 +00:00
f.Store.Shutdown()
2020-03-15 03:30:26 +00:00
}