2018-05-11 09:20:15 +00:00
|
|
|
package filer2
|
|
|
|
|
|
|
|
import (
|
2018-07-29 04:02:56 +00:00
|
|
|
"context"
|
2018-05-11 09:20:15 +00:00
|
|
|
"fmt"
|
2018-11-18 19:51:38 +00:00
|
|
|
"math"
|
2018-05-27 18:52:26 +00:00
|
|
|
"os"
|
2018-05-12 20:45:29 +00:00
|
|
|
"path/filepath"
|
2018-05-27 18:52:26 +00:00
|
|
|
"strings"
|
2018-05-12 20:45:29 +00:00
|
|
|
"time"
|
2018-07-29 04:02:56 +00:00
|
|
|
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2018-07-28 09:10:32 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
2018-07-29 04:02:56 +00:00
|
|
|
"github.com/karlseguin/ccache"
|
2018-05-11 09:20:15 +00:00
|
|
|
)
|
|
|
|
|
2018-12-03 03:42:50 +00:00
|
|
|
var (
|
|
|
|
OS_UID = uint32(os.Getuid())
|
|
|
|
OS_GID = uint32(os.Getgid())
|
|
|
|
)
|
|
|
|
|
2018-05-11 09:20:15 +00:00
|
|
|
type Filer struct {
|
2018-11-21 04:56:28 +00:00
|
|
|
store FilerStore
|
|
|
|
directoryCache *ccache.Cache
|
|
|
|
MasterClient *wdclient.MasterClient
|
|
|
|
fileIdDeletionChan chan string
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2018-06-01 07:39:39 +00:00
|
|
|
func NewFiler(masters []string) *Filer {
|
2018-11-21 04:56:28 +00:00
|
|
|
f := &Filer{
|
|
|
|
directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
|
|
|
|
MasterClient: wdclient.NewMasterClient(context.Background(), "filer", masters),
|
|
|
|
fileIdDeletionChan: make(chan string, 4096),
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
2018-11-21 04:56:28 +00:00
|
|
|
|
|
|
|
go f.loopProcessingDeletion()
|
|
|
|
|
|
|
|
return f
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2018-05-27 18:52:26 +00:00
|
|
|
func (f *Filer) SetStore(store FilerStore) {
|
2018-05-12 20:45:29 +00:00
|
|
|
f.store = store
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2018-05-27 18:52:26 +00:00
|
|
|
func (f *Filer) DisableDirectoryCache() {
|
2018-05-12 20:45:29 +00:00
|
|
|
f.directoryCache = nil
|
|
|
|
}
|
2018-05-11 09:20:15 +00:00
|
|
|
|
2018-07-28 09:10:32 +00:00
|
|
|
func (fs *Filer) GetMaster() string {
|
2018-07-28 21:22:46 +00:00
|
|
|
return fs.MasterClient.GetMaster()
|
2018-07-28 09:10:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (fs *Filer) KeepConnectedToMaster() {
|
2018-07-28 21:22:46 +00:00
|
|
|
fs.MasterClient.KeepConnectedToMaster()
|
2018-07-28 09:10:32 +00:00
|
|
|
}
|
|
|
|
|
2018-05-27 18:52:26 +00:00
|
|
|
func (f *Filer) CreateEntry(entry *Entry) error {
|
2018-05-12 20:45:29 +00:00
|
|
|
|
|
|
|
dirParts := strings.Split(string(entry.FullPath), "/")
|
|
|
|
|
|
|
|
// fmt.Printf("directory parts: %+v\n", dirParts)
|
|
|
|
|
|
|
|
var lastDirectoryEntry *Entry
|
|
|
|
|
|
|
|
for i := 1; i < len(dirParts); i++ {
|
|
|
|
dirPath := "/" + filepath.Join(dirParts[:i]...)
|
|
|
|
// fmt.Printf("%d directory: %+v\n", i, dirPath)
|
|
|
|
|
|
|
|
// first check local cache
|
|
|
|
dirEntry := f.cacheGetDirectory(dirPath)
|
|
|
|
|
|
|
|
// not found, check the store directly
|
|
|
|
if dirEntry == nil {
|
2018-05-19 19:07:15 +00:00
|
|
|
glog.V(4).Infof("find uncached directory: %s", dirPath)
|
2018-05-26 10:49:46 +00:00
|
|
|
dirEntry, _ = f.FindEntry(FullPath(dirPath))
|
2018-05-19 19:40:24 +00:00
|
|
|
} else {
|
2018-05-19 19:07:15 +00:00
|
|
|
glog.V(4).Infof("found cached directory: %s", dirPath)
|
2018-05-12 20:45:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// no such existing directory
|
2018-05-19 19:07:15 +00:00
|
|
|
if dirEntry == nil {
|
2018-05-12 20:45:29 +00:00
|
|
|
|
|
|
|
// create the directory
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
dirEntry = &Entry{
|
|
|
|
FullPath: FullPath(dirPath),
|
|
|
|
Attr: Attr{
|
|
|
|
Mtime: now,
|
|
|
|
Crtime: now,
|
2018-05-21 08:25:30 +00:00
|
|
|
Mode: os.ModeDir | 0770,
|
2018-05-12 20:45:29 +00:00
|
|
|
Uid: entry.Uid,
|
|
|
|
Gid: entry.Gid,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2018-05-21 08:25:30 +00:00
|
|
|
glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
|
2018-05-12 20:45:29 +00:00
|
|
|
mkdirErr := f.store.InsertEntry(dirEntry)
|
|
|
|
if mkdirErr != nil {
|
2018-09-30 07:49:52 +00:00
|
|
|
if _, err := f.FindEntry(FullPath(dirPath)); err == ErrNotFound {
|
|
|
|
return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
f.NotifyUpdateEvent(nil, dirEntry, false)
|
2018-05-12 20:45:29 +00:00
|
|
|
}
|
2018-08-13 08:20:49 +00:00
|
|
|
|
2018-07-12 04:53:31 +00:00
|
|
|
} else if !dirEntry.IsDirectory() {
|
|
|
|
return fmt.Errorf("%s is a file", dirPath)
|
2018-05-12 20:45:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// cache the directory entry
|
|
|
|
f.cacheSetDirectory(dirPath, dirEntry, i)
|
|
|
|
|
|
|
|
// remember the direct parent directory entry
|
|
|
|
if i == len(dirParts)-1 {
|
|
|
|
lastDirectoryEntry = dirEntry
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2018-05-11 09:20:15 +00:00
|
|
|
|
2018-05-12 20:45:29 +00:00
|
|
|
if lastDirectoryEntry == nil {
|
|
|
|
return fmt.Errorf("parent folder not found: %v", entry.FullPath)
|
|
|
|
}
|
|
|
|
|
2018-05-19 19:40:24 +00:00
|
|
|
/*
|
2018-05-27 18:52:26 +00:00
|
|
|
if !hasWritePermission(lastDirectoryEntry, entry) {
|
|
|
|
glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
|
|
|
|
lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
|
|
|
|
return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
|
|
|
|
}
|
2018-05-19 19:40:24 +00:00
|
|
|
*/
|
2018-05-12 20:45:29 +00:00
|
|
|
|
2018-06-17 20:02:22 +00:00
|
|
|
oldEntry, _ := f.FindEntry(entry.FullPath)
|
2018-05-31 03:24:57 +00:00
|
|
|
|
2018-09-22 07:11:46 +00:00
|
|
|
if oldEntry == nil {
|
|
|
|
if err := f.store.InsertEntry(entry); err != nil {
|
|
|
|
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
|
|
|
|
}
|
|
|
|
} else {
|
2018-11-29 08:07:54 +00:00
|
|
|
if err := f.UpdateEntry(oldEntry, entry); err != nil {
|
2018-09-22 07:11:46 +00:00
|
|
|
return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
|
|
|
|
}
|
2018-05-12 20:45:29 +00:00
|
|
|
}
|
|
|
|
|
2018-09-17 07:27:56 +00:00
|
|
|
f.NotifyUpdateEvent(oldEntry, entry, true)
|
2018-08-13 08:20:49 +00:00
|
|
|
|
2018-07-22 00:40:00 +00:00
|
|
|
f.deleteChunksIfNotNew(oldEntry, entry)
|
2018-06-17 20:02:22 +00:00
|
|
|
|
2018-05-12 20:45:29 +00:00
|
|
|
return nil
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2018-11-29 08:07:54 +00:00
|
|
|
func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
|
|
|
|
if oldEntry != nil {
|
|
|
|
if oldEntry.IsDirectory() && !entry.IsDirectory() {
|
|
|
|
return fmt.Errorf("existing %s is a directory", entry.FullPath)
|
|
|
|
}
|
|
|
|
if !oldEntry.IsDirectory() && entry.IsDirectory() {
|
|
|
|
return fmt.Errorf("existing %s is a file", entry.FullPath)
|
|
|
|
}
|
|
|
|
}
|
2018-05-22 10:26:38 +00:00
|
|
|
return f.store.UpdateEntry(entry)
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2018-05-26 10:49:46 +00:00
|
|
|
func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
|
2018-12-03 03:42:50 +00:00
|
|
|
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
if string(p) == "/" {
|
|
|
|
return &Entry{
|
|
|
|
FullPath: p,
|
|
|
|
Attr: Attr{
|
|
|
|
Mtime: now,
|
|
|
|
Crtime: now,
|
|
|
|
Mode: os.ModeDir | 0777,
|
|
|
|
Uid: OS_UID,
|
|
|
|
Gid: OS_GID,
|
|
|
|
},
|
|
|
|
}, nil
|
|
|
|
}
|
2018-05-11 09:20:15 +00:00
|
|
|
return f.store.FindEntry(p)
|
|
|
|
}
|
|
|
|
|
2018-07-19 08:21:44 +00:00
|
|
|
func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
|
2018-05-26 10:49:46 +00:00
|
|
|
entry, err := f.FindEntry(p)
|
|
|
|
if err != nil {
|
2018-05-31 03:24:57 +00:00
|
|
|
return err
|
2018-05-14 06:56:16 +00:00
|
|
|
}
|
2018-05-31 03:24:57 +00:00
|
|
|
|
2018-05-14 06:56:16 +00:00
|
|
|
if entry.IsDirectory() {
|
2018-08-01 08:26:41 +00:00
|
|
|
limit := int(1)
|
|
|
|
if isRecursive {
|
|
|
|
limit = math.MaxInt32
|
|
|
|
}
|
|
|
|
entries, err := f.ListDirectoryEntries(p, "", false, limit)
|
2018-05-14 06:56:16 +00:00
|
|
|
if err != nil {
|
2018-05-31 03:24:57 +00:00
|
|
|
return fmt.Errorf("list folder %s: %v", p, err)
|
2018-05-14 06:56:16 +00:00
|
|
|
}
|
2018-07-19 08:21:44 +00:00
|
|
|
if isRecursive {
|
|
|
|
for _, sub := range entries {
|
|
|
|
f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if len(entries) > 0 {
|
|
|
|
return fmt.Errorf("folder %s is not empty", p)
|
|
|
|
}
|
2018-05-14 06:56:16 +00:00
|
|
|
}
|
2018-07-12 01:27:25 +00:00
|
|
|
f.cacheDelDirectory(string(p))
|
2018-05-14 06:56:16 +00:00
|
|
|
}
|
2018-05-31 03:24:57 +00:00
|
|
|
|
2018-06-07 05:11:01 +00:00
|
|
|
if shouldDeleteChunks {
|
2018-10-14 07:12:28 +00:00
|
|
|
f.DeleteChunks(entry.Chunks)
|
2018-06-07 05:11:01 +00:00
|
|
|
}
|
2018-05-31 03:24:57 +00:00
|
|
|
|
2018-08-01 08:26:41 +00:00
|
|
|
if p == "/" {
|
|
|
|
return nil
|
|
|
|
}
|
2018-08-19 08:27:30 +00:00
|
|
|
glog.V(3).Infof("deleting entry %v", p)
|
2018-08-13 08:20:49 +00:00
|
|
|
|
2018-09-17 07:27:56 +00:00
|
|
|
f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
|
2018-08-13 08:20:49 +00:00
|
|
|
|
2018-05-11 09:20:15 +00:00
|
|
|
return f.store.DeleteEntry(p)
|
|
|
|
}
|
|
|
|
|
2018-05-13 21:02:29 +00:00
|
|
|
func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
|
2018-05-14 06:56:16 +00:00
|
|
|
if strings.HasSuffix(string(p), "/") && len(p) > 1 {
|
2018-07-22 00:40:00 +00:00
|
|
|
p = p[0 : len(p)-1]
|
2018-05-12 20:45:29 +00:00
|
|
|
}
|
2018-05-13 21:02:29 +00:00
|
|
|
return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2018-07-12 01:27:25 +00:00
|
|
|
func (f *Filer) cacheDelDirectory(dirpath string) {
|
|
|
|
if f.directoryCache == nil {
|
2018-07-19 08:21:44 +00:00
|
|
|
return
|
2018-07-12 01:27:25 +00:00
|
|
|
}
|
|
|
|
f.directoryCache.Delete(dirpath)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-05-27 18:52:26 +00:00
|
|
|
func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
|
2018-05-12 20:45:29 +00:00
|
|
|
if f.directoryCache == nil {
|
|
|
|
return nil
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
2018-05-12 20:45:29 +00:00
|
|
|
item := f.directoryCache.Get(dirpath)
|
|
|
|
if item == nil {
|
2018-05-11 09:20:15 +00:00
|
|
|
return nil
|
|
|
|
}
|
2018-05-12 20:45:29 +00:00
|
|
|
return item.Value().(*Entry)
|
|
|
|
}
|
2018-05-11 09:20:15 +00:00
|
|
|
|
2018-05-12 20:45:29 +00:00
|
|
|
func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
|
2018-05-11 09:20:15 +00:00
|
|
|
|
2018-05-12 20:45:29 +00:00
|
|
|
if f.directoryCache == nil {
|
|
|
|
return
|
2018-05-11 09:27:57 +00:00
|
|
|
}
|
|
|
|
|
2018-05-12 20:45:29 +00:00
|
|
|
minutes := 60
|
|
|
|
if level < 10 {
|
|
|
|
minutes -= level * 6
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2018-05-12 20:45:29 +00:00
|
|
|
f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
|
2018-05-11 09:20:15 +00:00
|
|
|
}
|