gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #645 from chrislusf/vasto_filer

Browse source 
merge back filer related refactoring
This commit is contained in:
Chris Lu 2018-05-05 14:12:51 -07:00 committed by GitHub
parent 94a35f25f3 7f49514f6f
commit 4d5e1e5947
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 166 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
weed/command/mount_std.go
View file

@ -59,27 +59,27 @@ func (WFS) Root() (fs.Node, error) {
var fileIdMap = make(map[uint64]filer.FileId)
type Dir struct {
Id uint64
Path string
DirentMap map[string]*fuse.Dirent
}
func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
attr.Inode = dir.Id
attr.Mode = os.ModeDir | 0555
return nil
}
func (dir *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
if dirent, ok := dir.DirentMap[name]; ok {
if dirent.Type == fuse.DT_File {
return &File{Id: dirent.Inode, FileId: fileIdMap[dirent.Inode], Name: dirent.Name}, nil
if entry, err := filer.LookupDirectoryEntry(*mountOptions.filer, dir.Path, name); err == nil {
if !entry.Found {
return nil, fuse.ENOENT
}
if entry.FileId != "" {
return &File{FileId: filer.FileId(entry.FileId), Name: name}, nil
} else {
return &Dir{Path: path.Join(dir.Path, name)}, nil
}
return &Dir{
Id: dirent.Inode,
Path: path.Join(dir.Path, dirent.Name),
}, nil
}
return nil, fuse.ENOENT
}
@ -90,17 +90,16 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
}
if dirs, e := filer.ListDirectories(*mountOptions.filer, dir.Path); e == nil {
for _, d := range dirs.Directories {
dirId := uint64(d.Id)
dirent := fuse.Dirent{Inode: dirId, Name: d.Name, Type: fuse.DT_Dir}
dirent := fuse.Dirent{Name: string(d), Type: fuse.DT_Dir}
ret = append(ret, dirent)
dir.DirentMap[d.Name] = &dirent
dir.DirentMap[string(d)] = &dirent
}
}
if files, e := filer.ListFiles(*mountOptions.filer, dir.Path, ""); e == nil {
for _, f := range files.Files {
if fileId, e := storage.ParseFileId(string(f.Id)); e == nil {
fileInode := uint64(fileId.VolumeId)<<48 + fileId.Key
dirent := fuse.Dirent{Inode: fileInode, Name: f.Name, Type: fuse.DT_File}
dirent := fuse.Dirent{Name: f.Name, Type: fuse.DT_File}
ret = append(ret, dirent)
dir.DirentMap[f.Name] = &dirent
fileIdMap[fileInode] = f.Id
@ -120,13 +119,11 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
}
type File struct {
Id uint64
FileId filer.FileId
Name string
}
func (file *File) Attr(context context.Context, attr *fuse.Attr) error {
attr.Inode = file.Id
attr.Mode = 0444
ret, err := filer.GetFileSize(*mountOptions.filer, string(file.FileId))
if err == nil {

19
weed/filer/client_operations.go
View file

@ -65,7 +65,7 @@ func GetFileContent(server string, fileId string) (ret *GetFileContentResult, er
}
type ListDirectoriesResult struct {
Directories []DirectoryEntry
Directories []DirectoryName
Error string `json:"error,omitempty"`
}
@ -80,6 +80,23 @@ func ListDirectories(server string, directory string) (ret *ListDirectoriesResul
return nil, err
}
type LookupDirectoryEntryResult struct {
Found bool
FileId string
Error string `json:"error,omitempty"`
}
func LookupDirectoryEntry(server string, directory string, name string) (ret *LookupDirectoryEntryResult, err error) {
ret = new(LookupDirectoryEntryResult)
if err := call(server, ApiRequest{Command: "lookupDirectoryEntry", Directory: directory, FileName: name}, ret); err == nil {
if ret.Error != "" {
return nil, errors.New(ret.Error)
}
return ret, nil
}
return nil, err
}
func DeleteDirectoryOrFile(server string, path string, isDir bool) error {
destUrl := fmt.Sprintf("http://%s%s", server, path)
if isDir {

8
weed/filer/embedded_filer/directory.go
View file

@ -5,11 +5,11 @@ import (
)
type DirectoryManager interface {
FindDirectory(dirPath string) (filer.DirectoryId, error)
ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
FindDirectory(dirPath string) (DirectoryId, error)
ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error)
MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error)
MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
DeleteDirectory(dirPath string) error
//functions used by FUSE
FindDirectoryById(filer.DirectoryId, error)
FindDirectoryById(DirectoryId, error)
}

22
weed/filer/embedded_filer/directory_in_map.go
View file

@ -16,12 +16,14 @@ import (
var writeLock sync.Mutex //serialize changes to dir.log
type DirectoryId int32
type DirectoryEntryInMap struct {
sync.Mutex
Name string
Parent *DirectoryEntryInMap
subDirectories map[string]*DirectoryEntryInMap
Id filer.DirectoryId
Id DirectoryId
}
func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
@ -45,18 +47,18 @@ func (de *DirectoryEntryInMap) hasChildren() bool {
defer de.Unlock()
return len(de.subDirectories) > 0
}
func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryName) {
de.Lock()
defer de.Unlock()
for k, v := range de.subDirectories {
dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
for k, _ := range de.subDirectories {
dirNames = append(dirNames, filer.DirectoryName(k))
}
return dirNames
}
type DirectoryManagerInMap struct {
Root *DirectoryEntryInMap
max filer.DirectoryId
max DirectoryId
logFile *os.File
isLoading bool
}
@ -113,7 +115,7 @@ func (dm *DirectoryManagerInMap) processEachLine(line string) error {
if pe != nil {
return pe
}
if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil {
return e
}
case "mov":
@ -172,7 +174,7 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI
}
return dir, nil
}
func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
func (dm *DirectoryManagerInMap) findDirectoryId(dirPath string) (DirectoryId, error) {
d, e := dm.findDirectory(dirPath)
if e == nil {
return d.Id, nil
@ -180,7 +182,7 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryI
return dm.Root.Id, e
}
func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error {
dirPath = CleanFilePath(dirPath)
if dirPath == "/" {
return nil
@ -248,7 +250,7 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn
return dir, created
}
func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) {
dir, _ := dm.makeDirectory(dirPath)
return dir.Id, nil
}
@ -275,7 +277,7 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent
return nil
}
func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryName, err error) {
d, e := dm.findDirectory(dirPath)
if e != nil {
return dirNames, e

8
weed/filer/embedded_filer/directory_test.go
View file

@ -19,17 +19,17 @@ func TestDirectory(t *testing.T) {
dm.MakeDirectory("/a/b/e/f")
dm.MakeDirectory("/a/b/e/f/g")
dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t")
if _, err := dm.FindDirectory("/a/b/e/f/g"); err == nil {
if _, err := dm.findDirectoryId("/a/b/e/f/g"); err == nil {
t.Fatal("/a/b/e/f/g should not exist any more after moving")
}
if _, err := dm.FindDirectory("/a/b/t"); err != nil {
if _, err := dm.findDirectoryId("/a/b/t"); err != nil {
t.Fatal("/a/b/t should exist after moving")
}
if _, err := dm.FindDirectory("/a/b/g"); err == nil {
if _, err := dm.findDirectoryId("/a/b/g"); err == nil {
t.Fatal("/a/b/g should not exist after moving")
}
dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "")
if _, err := dm.FindDirectory("/a/b/f"); err != nil {
if _, err := dm.findDirectoryId("/a/b/f"); err != nil {
t.Fatal("/a/b/g should not exist after moving")
}
dm.MakeDirectory("/a/b/g/h/i")

34
weed/filer/embedded_filer/filer_embedded.go
View file

@ -45,27 +45,37 @@ func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error)
}
func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
dir, file := filepath.Split(filePath)
dirId, e := filer.directories.FindDirectory(dir)
return filer.findFileEntry(dir, file)
}
func (filer *FilerEmbedded) findFileEntry(parentPath string, fileName string) (fid string, err error) {
dirId, e := filer.directories.findDirectoryId(parentPath)
if e != nil {
return "", e
}
return filer.files.FindFile(dirId, file)
return filer.files.FindFile(dirId, fileName)
}
func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
return filer.directories.FindDirectory(dirPath)
func (filer *FilerEmbedded) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
if _, err = filer.directories.findDirectory(filepath.Join(dirPath, name)); err == nil {
return true, "", nil
}
if fileId, err = filer.findFileEntry(dirPath, name); err == nil {
return true, fileId, nil
}
return false, "", err
}
func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
return filer.directories.ListDirectories(dirPath)
}
func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
dirId, e := filer.directories.FindDirectory(dirPath)
dirId, e := filer.directories.findDirectoryId(dirPath)
if e != nil {
return nil, e
}
return filer.files.ListFiles(dirId, lastFileName, limit), nil
}
func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) {
dirId, e := filer.directories.FindDirectory(dirPath)
dirId, e := filer.directories.findDirectoryId(dirPath)
if e != nil {
return e
}
@ -74,7 +84,7 @@ func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err
return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs))
}
for _, sub := range sub_dirs {
if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, sub.Name), recursive); delete_sub_err != nil {
if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, string(sub)), recursive); delete_sub_err != nil {
return delete_sub_err
}
}
@ -108,7 +118,7 @@ func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err
func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) {
dir, file := filepath.Split(filePath)
dirId, e := filer.directories.FindDirectory(dir)
dirId, e := filer.directories.findDirectoryId(dir)
if e != nil {
return "", e
}
@ -126,8 +136,8 @@ func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
filer.mvMutex.Lock()
defer filer.mvMutex.Unlock()
if _, dir_err := filer.FindDirectory(fromPath); dir_err == nil {
if _, err := filer.FindDirectory(toPath); err == nil {
if _, dir_err := filer.directories.findDirectoryId(fromPath); dir_err == nil {
if _, err := filer.directories.findDirectoryId(toPath); err == nil {
// move folder under an existing folder
return filer.directories.MoveUnderDirectory(fromPath, toPath, "")
}
@ -135,7 +145,7 @@ func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath))
}
if fid, file_err := filer.DeleteFile(fromPath); file_err == nil {
if _, err := filer.FindDirectory(toPath); err == nil {
if _, err := filer.directories.findDirectoryId(toPath); err == nil {
// move file under an existing folder
return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid)
}

10
weed/filer/embedded_filer/files_in_leveldb.go
View file

@ -28,7 +28,7 @@ func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
return
}
func genKey(dirId filer.DirectoryId, fileName string) []byte {
func genKey(dirId DirectoryId, fileName string) []byte {
ret := make([]byte, 0, 4+len(fileName))
for i := 3; i >= 0; i-- {
ret = append(ret, byte(dirId>>(uint(i)*8)))
@ -37,11 +37,11 @@ func genKey(dirId filer.DirectoryId, fileName string) []byte {
return ret
}
func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) {
glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
}
func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) {
if fid, err = fl.FindFile(dirId, fileName); err != nil {
if err == leveldb.ErrNotFound {
return "", nil
@ -51,7 +51,7 @@ func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string
err = fl.db.Delete(genKey(dirId, fileName), nil)
return fid, err
}
func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) {
data, e := fl.db.Get(genKey(dirId, fileName), nil)
if e == leveldb.ErrNotFound {
return "", filer.ErrNotFound
@ -60,7 +60,7 @@ func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string)
}
return string(data), nil
}
func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
dirKey := genKey(dirId, "")
iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)

11
weed/filer/filer.go
View file

@ -11,12 +11,7 @@ type FileEntry struct {
Id FileId `json:"fid,omitempty"`
}
type DirectoryId int32
type DirectoryEntry struct {
Name string //dir name without path
Id DirectoryId
}
type DirectoryName string
type Filer interface {
CreateFile(fullFileName string, fid string) (err error)
@ -24,11 +19,11 @@ type Filer interface {
DeleteFile(fullFileName string) (fid string, err error)
//Optional functions. embedded filer support these
FindDirectory(dirPath string) (dirId DirectoryId, err error)
ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
ListDirectories(dirPath string) (dirs []DirectoryName, err error)
ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
DeleteDirectory(dirPath string, recursive bool) (err error)
Move(fromPath string, toPath string) (err error)
LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error)
}
var ErrNotFound = errors.New("filer: no entry is found in filer store")

10
weed/filer/flat_namespace/flat_namespace_filer.go
View file

@ -4,6 +4,7 @@ import (
"errors"
"github.com/chrislusf/seaweedfs/weed/filer"
"path/filepath"
)
type FlatNamespaceFiler struct {
@ -28,10 +29,13 @@ func (filer *FlatNamespaceFiler) CreateFile(fullFileName string, fid string) (er
func (filer *FlatNamespaceFiler) FindFile(fullFileName string) (fid string, err error) {
return filer.store.Get(fullFileName)
}
func (filer *FlatNamespaceFiler) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
return 0, ErrNotImplemented
func (filer *FlatNamespaceFiler) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
if fileId, err = filer.FindFile(filepath.Join(dirPath, name)); err == nil {
return true, fileId, nil
}
return false, "", err
}
func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
return nil, ErrNotImplemented
}
func (filer *FlatNamespaceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {

14
weed/filer/postgres_store/postgres_native.go
View file

@ -14,6 +14,8 @@ import (
"strings"
)
type DirectoryId int32
func databaseExists(db *sql.DB, databaseName string) (bool, error) {
sqlStatement := "SELECT datname from pg_database WHERE datname='%s'"
row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName))
@ -293,13 +295,13 @@ func (s *PostgresStore) delete(uriPath string) error {
return nil
}
func (s *PostgresStore) lookupDirectory(dirPath string) (filer.DirectoryId, string, error) {
func (s *PostgresStore) lookupDirectory(dirPath string) (DirectoryId, string, error) {
directoryRoot, directoryName := s.mySplitPath(dirPath)
sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName)
var id filer.DirectoryId
var id DirectoryId
var dirRoot string
var dirName string
err := row.Scan(&id, &dirRoot, &dirName)
@ -312,7 +314,7 @@ func (s *PostgresStore) lookupDirectory(dirPath string) (filer.DirectoryId, stri
return id, filepath.Join(dirRoot, dirName), err
}
func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryEntry, err error) {
func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryName, err error) {
sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName)
rows, err := s.db.Query(sqlStatement, dirPath, limit)
@ -323,7 +325,7 @@ func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer
if rows != nil {
defer rows.Close()
for rows.Next() {
var id filer.DirectoryId
var id DirectoryId
var directoryRoot string
var directoryName string
@ -331,7 +333,7 @@ func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer
if scanErr != nil {
err = scanErr
}
dirs = append(dirs, filer.DirectoryEntry{Name: (directoryName), Id: id})
dirs = append(dirs, filer.DirectoryName(directoryName))
}
}
return
@ -344,7 +346,7 @@ func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bo
sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName)
row := s.db.QueryRow(sqlStatement, dirPath+"%")
var id filer.DirectoryId
var id DirectoryId
err := row.Scan(&id)
if err != nil {
if err == sql.ErrNoRows {

21
weed/filer/postgres_store/postgres_store.go
View file

@ -10,6 +10,7 @@ import (
_ "github.com/lib/pq"
_ "path/filepath"
"path/filepath"
)
const (
@ -74,6 +75,17 @@ func (s *PostgresStore) FindFile(fullFilePath string) (fid string, err error) {
return fid, err
}
func (s *PostgresStore) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
fullPath := filepath.Join(dirPath, name)
if fileId, err = s.FindFile(fullPath); err == nil {
return true, fileId, nil
}
if _, _, err := s.lookupDirectory(fullPath); err == nil {
return true, "", err
}
return false, "", err
}
func (s *PostgresStore) DeleteFile(fullFilePath string) (fid string, err error) {
if err != nil {
return "", fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
@ -90,14 +102,9 @@ func (s *PostgresStore) DeleteFile(fullFilePath string) (fid string, err error)
}
}
func (s *PostgresStore) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
dirId, _, err = s.lookupDirectory(dirPath)
return dirId, err
}
func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
dirs, err = s.findDirectories(dirPath, 20)
dirs, err = s.findDirectories(dirPath, 1000)
glog.V(3).Infof("Postgres ListDirs = found %d directories under %s", len(dirs), dirPath)

45
weed/filer/vasto_store/design.txt Normal file
View file

@ -0,0 +1,45 @@
There are two main components of a filer: directories and files.
My previous approach was to use some sequance number to generate directoryId.
However, this is not scalable. The id generation itself is a bottleneck.
It needs careful locking and deduplication checking to get a directoryId.
In a second design, each directory is deterministically mapped to UUID version 3,
which uses MD5 to map a tuple of <uuid, name> to a version 3 UUID.
However, this UUID3 approach is logically the same as storing the full path.
Storing the full path is the simplest design.
separator is a special byte, 0x00.
When writing a file:
<file parent full path, separator, file name> => fildId, file properties
For folders:
The filer breaks the directory path into folders.
for each folder:
if it is not in cache:
check whether the folder is created in the KVS, if not:
set <folder parent full path, separator, folder name> => directory properties
if no permission for the folder:
break
The filer caches the most recently used folder permissions with a TTL.
So any folder permission change needs to wait TTL interval to take effect.
When listing the directory:
prefix scan of using (the folder full path + separator) as the prefix
The downside:
1. Rename a folder will need to recursively process all sub folders and files.
2. Move a folder will need to recursively process all sub folders and files.
So these operations are not allowed if the folder is not empty.
Allowing:
1. Rename a file
2. Move a file to a different folder
3. Delete an empty folder

9
weed/server/filer_server_handlers_api.go
View file

@ -21,6 +21,13 @@ func (fs *FilerServer) apiHandler(w http.ResponseWriter, r *http.Request) {
writeJsonError(w, r, http.StatusInternalServerError, err)
}
switch apiRequest.Command {
case "lookupDirectoryEntry":
res := filer.LookupDirectoryEntryResult{}
res.Found, res.FileId, err = fs.filer.LookupDirectoryEntry(apiRequest.Directory, apiRequest.FileName)
if err != nil {
res.Error = err.Error()
}
writeJsonQuiet(w, r, http.StatusOK, res)
case "listDirectories":
res := filer.ListDirectoriesResult{}
res.Directories, err = fs.filer.ListDirectories(apiRequest.Directory)
@ -30,7 +37,7 @@ func (fs *FilerServer) apiHandler(w http.ResponseWriter, r *http.Request) {
writeJsonQuiet(w, r, http.StatusOK, res)
case "listFiles":
res := filer.ListFilesResult{}
res.Files, err = fs.filer.ListFiles(apiRequest.Directory, apiRequest.FileName, 100)
res.Files, err = fs.filer.ListFiles(apiRequest.Directory, apiRequest.FileName, 1000)
if err != nil {
res.Error = err.Error()
}

6
weed/server/filer_ui/templates.go
View file

@ -28,15 +28,17 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
{{$path := .Path }}
{{ range $dirs_index, $dir := .Directories }}
<li>
<a href= {{ print $path $dir.Name "/"}} >
{{ $dir.Name }}
<a href={{ print $path $dir "/"}} >
{{ $dir }}
</a>
</li>
{{ end }}
{{ range $file_index, $file := .Files }}
<li>
<a href={{ print $path $file.Name}} >
{{ $file.Name }}
</a>
</li>
{{ end }}
</ul>