From 81af1bafbaf556fa234e5e26d197e8ed44aa4026 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 12 May 2018 13:45:29 -0700 Subject: [PATCH] file store adds memdb implementation --- weed/filer2/embedded/embedded_store.go | 21 +++- weed/filer2/filer.go | 158 +++++++++++++++++-------- weed/filer2/filer_structure.go | 53 +++++---- weed/filer2/filer_test.go | 30 ----- weed/filer2/memdb/memdb_store.go | 94 +++++++++++++++ weed/filer2/memdb/memdb_store_test.go | 96 +++++++++++++++ weed/filer2/permission.go | 22 ++++ 7 files changed, 368 insertions(+), 106 deletions(-) delete mode 100644 weed/filer2/filer_test.go create mode 100644 weed/filer2/memdb/memdb_store.go create mode 100644 weed/filer2/memdb/memdb_store_test.go create mode 100644 weed/filer2/permission.go diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index a0f7f0385..a2a45807a 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -2,6 +2,7 @@ package embedded import ( "github.com/syndtr/goleveldb/leveldb" + "github.com/chrislusf/seaweedfs/weed/filer2" ) type EmbeddedStore struct { @@ -16,10 +17,26 @@ func NewEmbeddedStore(dir string) (filer *EmbeddedStore, err error) { return } -func (filer *EmbeddedStore) CreateFile(filePath string, fid string) (err error) { +func (filer *EmbeddedStore) InsertEntry(entry *filer2.Entry) (err error) { return nil } -func (filer *EmbeddedStore) Mkdir(filePath string) (err error) { +func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int32) (err error) { return nil } + +func (filer *EmbeddedStore) AppendFileChunk(fullpath filer2.FullPath, fileChunk filer2.FileChunk) (err error) { + return nil +} + +func (filer *EmbeddedStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) { + return false, nil, nil +} + +func (filer *EmbeddedStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + return nil, nil +} + +func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) { + return nil, nil +} diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 0592c7848..848e87b2d 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -3,9 +3,11 @@ package filer2 import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2/embedded" "github.com/karlseguin/ccache" "strings" + "path/filepath" + "time" + "os" ) type Filer struct { @@ -21,78 +23,132 @@ func NewFiler(master string) *Filer { } } -func NewEmbeddedFiler(master string, dir string) (*Filer, error) { - _, err := embedded.NewEmbeddedStore(dir) - if err != nil { - return nil, fmt.Errorf("failed to create embedded filer store: %v", err) - } - return &Filer{ - master: master, - // store: store, - }, nil +func (f *Filer) SetStore(store FilerStore) () { + f.store = store } -func (f *Filer) CreateEntry(entry Entry) (error) { - /* - 1. recursively ensure parent directory is created. - 2. get current parent directory, add link, - 3. add the file entry - */ +func (f *Filer) DisableDirectoryCache() () { + f.directoryCache = nil +} - recursivelyEnsureDirectory(entry.Dir, func(parent, name string) error { - return nil - }) +func (f *Filer) CreateEntry(entry *Entry) (error) { - return f.store.CreateEntry(entry) + dirParts := strings.Split(string(entry.FullPath), "/") + + // fmt.Printf("directory parts: %+v\n", dirParts) + + var lastDirectoryEntry *Entry + + for i := 1; i < len(dirParts); i++ { + dirPath := "/" + filepath.Join(dirParts[:i]...) + // fmt.Printf("%d directory: %+v\n", i, dirPath) + + dirFound := false + + // first check local cache + dirEntry := f.cacheGetDirectory(dirPath) + + // not found, check the store directly + if dirEntry == nil { + var dirFindErr error + dirFound, dirEntry, dirFindErr = f.FindEntry(FullPath(dirPath)) + if dirFindErr != nil { + return fmt.Errorf("findDirectory %s: %v", dirPath, dirFindErr) + } + } + + // no such existing directory + if !dirFound { + + // create the directory + now := time.Now() + + dirEntry = &Entry{ + FullPath: FullPath(dirPath), + Attr: Attr{ + Mtime: now, + Crtime: now, + Mode: os.ModeDir | 0660, + Uid: entry.Uid, + Gid: entry.Gid, + Nlink: 2, + }, + } + + mkdirErr := f.store.InsertEntry(dirEntry) + if mkdirErr != nil { + return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr) + } + } + + // cache the directory entry + f.cacheSetDirectory(dirPath, dirEntry, i) + + // remember the direct parent directory entry + if i == len(dirParts)-1 { + lastDirectoryEntry = dirEntry + } + + } + + if lastDirectoryEntry == nil { + return fmt.Errorf("parent folder not found: %v", entry.FullPath) + } + + if !hasWritePermission(lastDirectoryEntry, entry) { + return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath) + } + + if err := f.store.InsertEntry(entry); err != nil { + return fmt.Errorf("insert entry %s: %v", entry.FullPath, err) + } + if err := f.store.AddDirectoryLink(lastDirectoryEntry, 1); err != nil { + return fmt.Errorf("insert entry %s: %v", entry.FullPath, err) + } + + return nil } func (f *Filer) AppendFileChunk(p FullPath, c FileChunk) (err error) { return f.store.AppendFileChunk(p, c) } -func (f *Filer) FindEntry(p FullPath) (found bool, fileEntry Entry, err error) { +func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { return f.store.FindEntry(p) } -func (f *Filer) DeleteEntry(p FullPath) (fileEntry Entry, err error) { +func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) { return f.store.DeleteEntry(p) } -func (f *Filer) ListDirectoryEntries(p FullPath) ([]Entry, error) { +func (f *Filer) ListDirectoryEntries(p FullPath) ([]*Entry, error) { + if strings.HasSuffix(string(p), "/") { + p = p[0:len(p)-1] + } return f.store.ListDirectoryEntries(p) } -func (f *Filer) UpdateEntry(entry Entry) (error) { - return f.store.UpdateEntry(entry) -} - -func recursivelyEnsureDirectory(fullPath string, fn func(parent, name string) error) (error) { - if strings.HasSuffix(fullPath, "/") { - fullPath = fullPath[0:len(fullPath)-1] - } - nextPathEnd := strings.LastIndex(fullPath, "/") - if nextPathEnd < 0 { +func (f *Filer) cacheGetDirectory(dirpath string) (*Entry) { + if f.directoryCache == nil { return nil } - - dirName := fullPath[nextPathEnd+1:] - parentDirPath := fullPath[0:nextPathEnd] - - if parentDirPath == "" { - parentDirPath = "/" + item := f.directoryCache.Get(dirpath) + if item == nil { + return nil } - - if err := recursivelyEnsureDirectory(parentDirPath, fn); err != nil { - return err - } - - if err := fn(parentDirPath, dirName); err != nil { - return err - } - - return nil + return item.Value().(*Entry) } -func (f *Filer) cacheGetDirectory(dirpath string) (error) { - return nil +func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) { + + if f.directoryCache == nil { + return + } + + minutes := 60 + if level < 10 { + minutes -= level * 6 + } + + f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute) } diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 603645e34..2aa3d44d2 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -4,28 +4,35 @@ import ( "errors" "os" "time" + "path/filepath" ) type FileId string //file id in SeaweedFS -type FullPath struct { - Dir string //full path of the parent dir - Name string //file name without path +type FullPath string + +func (fp FullPath) DirAndName() (string, string) { + dir, name := filepath.Split(string(fp)) + if dir == "/" { + return dir, name + } + if len(dir) < 1 { + return "/", "" + } + return dir[:len(dir)-1], name } type Attr struct { - Mtime time.Time // time of last modification - Crtime time.Time // time of creation (OS X only) - Mode os.FileMode // file mode - Uid uint32 // owner uid - Gid uint32 // group gid - IsDirectory bool - Size uint64 // total size in bytes - Nlink uint32 // number of links (usually 1) + Mtime time.Time // time of last modification + Crtime time.Time // time of creation (OS X only) + Mode os.FileMode // file mode + Uid uint32 // owner uid + Gid uint32 // group gid + Size uint64 // total size in bytes + Nlink uint32 // number of links (usually 1) } type Entry struct { - Dir string `json:"dir,omitempty"` //full path of the parent dir - Name string `json:"name,omitempty"` //file name without path + FullPath Attr @@ -40,23 +47,23 @@ type FileChunk struct { } type AbstractFiler interface { - CreateEntry(Entry) (error) + CreateEntry(*Entry) (error) AppendFileChunk(FullPath, FileChunk) (err error) - FindEntry(FullPath) (found bool, fileEntry Entry, err error) - DeleteEntry(FullPath) (fileEntry Entry, err error) + FindEntry(FullPath) (found bool, fileEntry *Entry, err error) + DeleteEntry(FullPath) (fileEntry *Entry, err error) - ListDirectoryEntries(dirPath FullPath) ([]Entry, error) - UpdateEntry(Entry) (error) + ListDirectoryEntries(dirPath FullPath) ([]*Entry, error) + UpdateEntry(*Entry) (error) } var ErrNotFound = errors.New("filer: no entry is found in filer store") type FilerStore interface { - CreateEntry(Entry) (error) + InsertEntry(*Entry) (error) + AddDirectoryLink(directory *Entry, delta int32) (err error) AppendFileChunk(FullPath, FileChunk) (err error) - FindEntry(FullPath) (found bool, fileEntry Entry, err error) - DeleteEntry(FullPath) (fileEntry Entry, err error) + FindEntry(FullPath) (found bool, entry *Entry, err error) + DeleteEntry(FullPath) (fileEntry *Entry, err error) - ListDirectoryEntries(dirPath FullPath) ([]Entry, error) - UpdateEntry(Entry) (error) + ListDirectoryEntries(dirPath FullPath) ([]*Entry, error) } diff --git a/weed/filer2/filer_test.go b/weed/filer2/filer_test.go deleted file mode 100644 index bafc25cf0..000000000 --- a/weed/filer2/filer_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package filer2 - -import ( - "testing" - "path/filepath" - "fmt" -) - -func TestRecursion(t *testing.T) { - fullPath := "/home/chris/some/file/abc.jpg" - expected := []string{ - "/", "home", - "/home", "chris", - "/home/chris", "some", - "/home/chris/some", "file", - } - - dir, _ := filepath.Split(fullPath) - - i := 0 - - recursivelyEnsureDirectory(dir, func(parent, name string) error { - if parent != expected[i] || name != expected[i+1] { - t.Errorf("recursive directory is wrong! parent=%s dirName=%s", parent, name) - } - fmt.Printf("processing folder %s \t parent=%s\n", name, parent) - i += 2 - return nil - }) -} diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go new file mode 100644 index 000000000..6925aacfb --- /dev/null +++ b/weed/filer2/memdb/memdb_store.go @@ -0,0 +1,94 @@ +package memdb + +import ( + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/google/btree" + "strings" + "fmt" +) + +type MemDbStore struct { + tree *btree.BTree +} + +type Entry struct { + *filer2.Entry +} + +func (a Entry) Less(b btree.Item) bool { + return strings.Compare(string(a.FullPath), string(b.(Entry).FullPath)) < 0 +} + +func NewMemDbStore() (filer *MemDbStore) { + filer = &MemDbStore{} + filer.tree = btree.New(8) + return +} + +func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { + // println("inserting", entry.FullPath) + filer.tree.ReplaceOrInsert(Entry{entry}) + return nil +} + +func (filer *MemDbStore) AddDirectoryLink(directory *filer2.Entry, delta int32) (err error) { + directory.Nlink = uint32(int32(directory.Nlink) + delta) + return nil +} + +func (filer *MemDbStore) AppendFileChunk(fullpath filer2.FullPath, fileChunk filer2.FileChunk) (err error) { + found, entry, err := filer.FindEntry(fullpath) + if !found { + return fmt.Errorf("No such file: %s", fullpath) + } + entry.Chunks = append(entry.Chunks, fileChunk) + return nil +} + +func (filer *MemDbStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) { + item := filer.tree.Get(Entry{&filer2.Entry{FullPath: fullpath}}) + if item == nil { + return false, nil, nil + } + entry = item.(Entry).Entry + return true, entry, nil +} + +func (filer *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + item := filer.tree.Delete(Entry{&filer2.Entry{FullPath: fullpath}}) + if item == nil { + return nil, nil + } + entry = item.(Entry).Entry + return entry, nil +} + +func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) { + filer.tree.AscendGreaterOrEqual(Entry{&filer2.Entry{FullPath: fullpath}}, + func(item btree.Item) bool { + entry := item.(Entry).Entry + // println("checking", entry.FullPath) + if entry.FullPath == fullpath { + // skipping the current directory + // println("skipping the folder", entry.FullPath) + return true + } + dir, _ := entry.FullPath.DirAndName() + if !strings.HasPrefix(dir, string(fullpath)) { + // println("directory is:", dir, "fullpath:", fullpath) + // println("breaking from", entry.FullPath) + return false + } + if dir != string(fullpath) { + // this could be items in deeper directories + // println("skipping deeper folder", entry.FullPath) + return true + } + // now process the directory items + // println("adding entry", entry.FullPath) + entries = append(entries, entry) + return true + }, + ) + return entries, nil +} diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go new file mode 100644 index 000000000..7e515d9f5 --- /dev/null +++ b/weed/filer2/memdb/memdb_store_test.go @@ -0,0 +1,96 @@ +package memdb + +import ( + "testing" + "github.com/chrislusf/seaweedfs/weed/filer2" +) + +func TestCreateAndFind(t *testing.T) { + filer := filer2.NewFiler("") + filer.SetStore(NewMemDbStore()) + filer.DisableDirectoryCache() + + fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg") + + entry1 := &filer2.Entry{ + FullPath: fullpath, + Attr: filer2.Attr{ + Mode: 0440, + Uid: 1234, + Gid: 5678, + }, + } + + if err := filer.CreateEntry(entry1); err != nil { + t.Errorf("create entry %v: %v", entry1.FullPath, err) + return + } + + found, entry, err := filer.FindEntry(fullpath) + + if err != nil { + t.Errorf("find entry: %v", err) + return + } + + if !found { + t.Errorf("Failed to find newly created file") + return + } + + if entry.FullPath != entry1.FullPath { + t.Errorf("find wrong entry: %v", entry.FullPath) + return + } + +} + +func TestCreateFileAndList(t *testing.T) { + filer := filer2.NewFiler("") + filer.SetStore(NewMemDbStore()) + filer.DisableDirectoryCache() + + entry1 := &filer2.Entry{ + FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"), + Attr: filer2.Attr{ + Mode: 0440, + Uid: 1234, + Gid: 5678, + }, + } + + entry2 := &filer2.Entry{ + FullPath: filer2.FullPath("/home/chris/this/is/one/file2.jpg"), + Attr: filer2.Attr{ + Mode: 0440, + Uid: 1234, + Gid: 5678, + }, + } + + filer.CreateEntry(entry1) + filer.CreateEntry(entry2) + + entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/")) + + if err != nil { + t.Errorf("list entries: %v", err) + return + } + + if len(entries) != 2 { + t.Errorf("list entries count: %v", len(entries)) + return + } + + if entries[0].FullPath != entry1.FullPath { + t.Errorf("find wrong entry 1: %v", entries[0].FullPath) + return + } + + if entries[1].FullPath != entry2.FullPath { + t.Errorf("find wrong entry 2: %v", entries[1].FullPath) + return + } + +} diff --git a/weed/filer2/permission.go b/weed/filer2/permission.go new file mode 100644 index 000000000..8a9508fbc --- /dev/null +++ b/weed/filer2/permission.go @@ -0,0 +1,22 @@ +package filer2 + +func hasWritePermission(dir *Entry, entry *Entry) bool { + + if dir == nil { + return false + } + + if dir.Uid == entry.Uid && dir.Mode&0200 > 0 { + return true + } + + if dir.Gid == entry.Gid && dir.Mode&0020 > 0 { + return true + } + + if dir.Mode&0002 > 0 { + return true + } + + return false +}