gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

remove directory cache

Browse source
This commit is contained in:
Chris Lu 2020-08-12 13:11:04 -07:00
parent 83cad3da79
commit 8824a9755c
4 changed files with 4 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
weed/filer2/filer.go
View file

@ -9,8 +9,6 @@ import (
"google.golang.org/grpc"
"github.com/karlseguin/ccache"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
@ -27,7 +25,6 @@ var (
type Filer struct {
Store *FilerStoreWrapper
directoryCache *ccache.Cache
MasterClient *wdclient.MasterClient
fileIdDeletionQueue *util.UnboundedQueue
GrpcDialOption grpc.DialOption
@ -44,7 +41,6 @@ type Filer struct {
func NewFiler(masters []string, grpcDialOption grpc.DialOption,
filerHost string, filerGrpcPort uint32, collection string, replication string, notifyFn func()) *Filer {
f := &Filer{
directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
MasterClient: wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, filerGrpcPort, masters),
fileIdDeletionQueue: util.NewUnboundedQueue(),
GrpcDialOption: grpcDialOption,
@ -77,10 +73,6 @@ func (f *Filer) GetStore() (store FilerStore) {
return f.Store
}
func (f *Filer) DisableDirectoryCache() {
f.directoryCache = nil
}
func (fs *Filer) GetMaster() string {
return fs.MasterClient.GetMaster()
}
@ -117,16 +109,9 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
dirPath := "/" + util.Join(dirParts[:i]...)
// fmt.Printf("%d directory: %+v\n", i, dirPath)
// first check local cache
dirEntry := f.cacheGetDirectory(dirPath)
// not found, check the store directly
if dirEntry == nil {
// check the store directly, skipping cached directories
glog.V(4).Infof("find uncached directory: %s", dirPath)
dirEntry, _ = f.FindEntry(ctx, util.FullPath(dirPath))
} else {
// glog.V(4).Infof("found cached directory: %s", dirPath)
}
dirEntry, _ := f.FindEntry(ctx, util.FullPath(dirPath))
// no such existing directory
if dirEntry == nil {
@ -166,9 +151,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
return fmt.Errorf("%s is a file", dirPath)
}
// cache the directory entry
f.cacheSetDirectory(dirPath, dirEntry, i)
// remember the direct parent directory entry
if i == len(dirParts)-1 {
lastDirectoryEntry = dirEntry
@ -295,45 +277,6 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta
return
}
func (f *Filer) cacheDelDirectory(dirpath string) {
if dirpath == "/" {
return
}
if f.directoryCache == nil {
return
}
f.directoryCache.Delete(dirpath)
return
}
func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
if f.directoryCache == nil {
return nil
}
item := f.directoryCache.Get(dirpath)
if item == nil {
return nil
}
return item.Value().(*Entry)
}
func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
if f.directoryCache == nil {
return
}
minutes := 60
if level < 10 {
minutes -= level * 6
}
f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
}
func (f *Filer) Shutdown() {
f.LocalMetaLogBuffer.Shutdown()
f.Store.Shutdown()

5
weed/filer2/filer_delete_entry.go
View file

@ -73,7 +73,6 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
var dirChunks []*filer_pb.FileChunk
if sub.IsDirectory() {
dirChunks, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, false)
f.cacheDelDirectory(string(sub.FullPath))
chunks = append(chunks, dirChunks...)
} else {
f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster)
@ -107,9 +106,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
if storeDeletionErr := f.Store.DeleteEntry(ctx, entry.FullPath); storeDeletionErr != nil {
return fmt.Errorf("filer store delete: %v", storeDeletionErr)
}
if entry.IsDirectory() {
f.cacheDelDirectory(string(entry.FullPath))
} else {
if !entry.IsDirectory() {
f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster)
}

2
weed/filer2/leveldb/leveldb_store_test.go
View file

@ -17,7 +17,6 @@ func TestCreateAndFind(t *testing.T) {
store := &LevelDBStore{}
store.initialize(dir)
filer.SetStore(store)
filer.DisableDirectoryCache()
fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg")
@ -72,7 +71,6 @@ func TestEmptyRoot(t *testing.T) {
store := &LevelDBStore{}
store.initialize(dir)
filer.SetStore(store)
filer.DisableDirectoryCache()
ctx := context.Background()

2
weed/filer2/leveldb2/leveldb2_store_test.go
View file

@ -17,7 +17,6 @@ func TestCreateAndFind(t *testing.T) {
store := &LevelDB2Store{}
store.initialize(dir, 2)
filer.SetStore(store)
filer.DisableDirectoryCache()
fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg")
@ -72,7 +71,6 @@ func TestEmptyRoot(t *testing.T) {
store := &LevelDB2Store{}
store.initialize(dir, 2)
filer.SetStore(store)
filer.DisableDirectoryCache()
ctx := context.Background()