seaweedfs/weed/filer2/filer.go

271 lines
6 KiB
Go
Raw Normal View History

2018-05-11 09:20:15 +00:00
package filer2
import (
2018-07-29 04:02:56 +00:00
"context"
2018-05-11 09:20:15 +00:00
"fmt"
2018-05-27 18:52:26 +00:00
"os"
2018-05-12 20:45:29 +00:00
"path/filepath"
2018-05-27 18:52:26 +00:00
"strings"
2018-05-12 20:45:29 +00:00
"time"
2018-07-29 04:02:56 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
2018-07-28 09:10:32 +00:00
"github.com/chrislusf/seaweedfs/weed/wdclient"
2018-07-29 04:02:56 +00:00
"github.com/karlseguin/ccache"
2018-08-01 08:26:41 +00:00
"math"
2018-05-11 09:20:15 +00:00
)
type Filer struct {
store FilerStore
directoryCache *ccache.Cache
MasterClient *wdclient.MasterClient
2018-05-11 09:20:15 +00:00
}
func NewFiler(masters []string) *Filer {
2018-05-11 09:20:15 +00:00
return &Filer{
directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
MasterClient: wdclient.NewMasterClient(context.Background(), "filer", masters),
2018-05-11 09:20:15 +00:00
}
}
2018-05-27 18:52:26 +00:00
func (f *Filer) SetStore(store FilerStore) {
2018-05-12 20:45:29 +00:00
f.store = store
2018-05-11 09:20:15 +00:00
}
2018-05-27 18:52:26 +00:00
func (f *Filer) DisableDirectoryCache() {
2018-05-12 20:45:29 +00:00
f.directoryCache = nil
}
2018-05-11 09:20:15 +00:00
2018-07-28 09:10:32 +00:00
func (fs *Filer) GetMaster() string {
return fs.MasterClient.GetMaster()
2018-07-28 09:10:32 +00:00
}
func (fs *Filer) KeepConnectedToMaster() {
fs.MasterClient.KeepConnectedToMaster()
2018-07-28 09:10:32 +00:00
}
2018-05-27 18:52:26 +00:00
func (f *Filer) CreateEntry(entry *Entry) error {
2018-05-12 20:45:29 +00:00
dirParts := strings.Split(string(entry.FullPath), "/")
// fmt.Printf("directory parts: %+v\n", dirParts)
var lastDirectoryEntry *Entry
for i := 1; i < len(dirParts); i++ {
dirPath := "/" + filepath.Join(dirParts[:i]...)
// fmt.Printf("%d directory: %+v\n", i, dirPath)
// first check local cache
dirEntry := f.cacheGetDirectory(dirPath)
// not found, check the store directly
if dirEntry == nil {
glog.V(4).Infof("find uncached directory: %s", dirPath)
dirEntry, _ = f.FindEntry(FullPath(dirPath))
} else {
glog.V(4).Infof("found cached directory: %s", dirPath)
2018-05-12 20:45:29 +00:00
}
// no such existing directory
if dirEntry == nil {
2018-05-12 20:45:29 +00:00
// create the directory
now := time.Now()
dirEntry = &Entry{
FullPath: FullPath(dirPath),
Attr: Attr{
Mtime: now,
Crtime: now,
2018-05-21 08:25:30 +00:00
Mode: os.ModeDir | 0770,
2018-05-12 20:45:29 +00:00
Uid: entry.Uid,
Gid: entry.Gid,
},
}
2018-05-21 08:25:30 +00:00
glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
2018-05-12 20:45:29 +00:00
mkdirErr := f.store.InsertEntry(dirEntry)
if mkdirErr != nil {
return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
}
2018-08-13 08:20:49 +00:00
2018-09-17 07:27:56 +00:00
f.NotifyUpdateEvent(nil, dirEntry, false)
2018-08-13 08:20:49 +00:00
} else if !dirEntry.IsDirectory() {
return fmt.Errorf("%s is a file", dirPath)
2018-05-12 20:45:29 +00:00
}
// cache the directory entry
f.cacheSetDirectory(dirPath, dirEntry, i)
// remember the direct parent directory entry
if i == len(dirParts)-1 {
lastDirectoryEntry = dirEntry
}
}
2018-05-11 09:20:15 +00:00
2018-05-12 20:45:29 +00:00
if lastDirectoryEntry == nil {
return fmt.Errorf("parent folder not found: %v", entry.FullPath)
}
/*
2018-05-27 18:52:26 +00:00
if !hasWritePermission(lastDirectoryEntry, entry) {
glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
}
*/
2018-05-12 20:45:29 +00:00
oldEntry, _ := f.FindEntry(entry.FullPath)
if oldEntry == nil {
if err := f.store.InsertEntry(entry); err != nil {
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
}
} else {
if err := f.store.UpdateEntry(entry); err != nil {
return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
}
2018-05-12 20:45:29 +00:00
}
2018-09-17 07:27:56 +00:00
f.NotifyUpdateEvent(oldEntry, entry, true)
2018-08-13 08:20:49 +00:00
f.deleteChunksIfNotNew(oldEntry, entry)
2018-05-12 20:45:29 +00:00
return nil
2018-05-11 09:20:15 +00:00
}
func (f *Filer) UpdateEntry(entry *Entry) (err error) {
return f.store.UpdateEntry(entry)
2018-05-11 09:20:15 +00:00
}
func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
2018-05-11 09:20:15 +00:00
return f.store.FindEntry(p)
}
func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
entry, err := f.FindEntry(p)
if err != nil {
return err
2018-05-14 06:56:16 +00:00
}
2018-05-14 06:56:16 +00:00
if entry.IsDirectory() {
2018-08-01 08:26:41 +00:00
limit := int(1)
if isRecursive {
limit = math.MaxInt32
}
entries, err := f.ListDirectoryEntries(p, "", false, limit)
2018-05-14 06:56:16 +00:00
if err != nil {
return fmt.Errorf("list folder %s: %v", p, err)
2018-05-14 06:56:16 +00:00
}
if isRecursive {
for _, sub := range entries {
f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
}
} else {
if len(entries) > 0 {
return fmt.Errorf("folder %s is not empty", p)
}
2018-05-14 06:56:16 +00:00
}
f.cacheDelDirectory(string(p))
2018-05-14 06:56:16 +00:00
}
2018-06-07 05:11:01 +00:00
if shouldDeleteChunks {
f.deleteChunks(entry.Chunks)
2018-06-07 05:11:01 +00:00
}
2018-08-01 08:26:41 +00:00
if p == "/" {
return nil
}
2018-08-19 08:27:30 +00:00
glog.V(3).Infof("deleting entry %v", p)
2018-08-13 08:20:49 +00:00
2018-09-17 07:27:56 +00:00
f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
2018-08-13 08:20:49 +00:00
2018-05-11 09:20:15 +00:00
return f.store.DeleteEntry(p)
}
2018-05-13 21:02:29 +00:00
func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
2018-05-14 06:56:16 +00:00
if strings.HasSuffix(string(p), "/") && len(p) > 1 {
p = p[0 : len(p)-1]
2018-05-12 20:45:29 +00:00
}
2018-05-13 21:02:29 +00:00
return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
2018-05-11 09:20:15 +00:00
}
func (f *Filer) cacheDelDirectory(dirpath string) {
if f.directoryCache == nil {
return
}
f.directoryCache.Delete(dirpath)
return
}
2018-05-27 18:52:26 +00:00
func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
2018-05-12 20:45:29 +00:00
if f.directoryCache == nil {
return nil
2018-05-11 09:20:15 +00:00
}
2018-05-12 20:45:29 +00:00
item := f.directoryCache.Get(dirpath)
if item == nil {
2018-05-11 09:20:15 +00:00
return nil
}
2018-05-12 20:45:29 +00:00
return item.Value().(*Entry)
}
2018-05-11 09:20:15 +00:00
2018-05-12 20:45:29 +00:00
func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
2018-05-11 09:20:15 +00:00
2018-05-12 20:45:29 +00:00
if f.directoryCache == nil {
return
2018-05-11 09:27:57 +00:00
}
2018-05-12 20:45:29 +00:00
minutes := 60
if level < 10 {
minutes -= level * 6
2018-05-11 09:20:15 +00:00
}
2018-05-12 20:45:29 +00:00
f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
2018-05-11 09:20:15 +00:00
}
func (f *Filer) deleteChunks(chunks []*filer_pb.FileChunk) {
for _, chunk := range chunks {
f.DeleteFileByFileId(chunk.FileId)
}
}
func (f *Filer) DeleteFileByFileId(fileId string) {
fileUrlOnVolume, err := f.MasterClient.LookupFileId(fileId)
if err != nil {
glog.V(0).Infof("can not find file %s: %v", fileId, err)
}
if err := operation.DeleteFromVolumeServer(fileUrlOnVolume, ""); err != nil {
glog.V(0).Infof("deleting file %s: %v", fileId, err)
}
}
func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
if oldEntry == nil {
return
}
if newEntry == nil {
f.deleteChunks(oldEntry.Chunks)
}
var toDelete []*filer_pb.FileChunk
for _, oldChunk := range oldEntry.Chunks {
found := false
for _, newChunk := range newEntry.Chunks {
if oldChunk.FileId == newChunk.FileId {
found = true
break
}
}
if !found {
toDelete = append(toDelete, oldChunk)
}
}
f.deleteChunks(toDelete)
}