From 050ab1926409edbb8ac1186b1fb83e80c17fc33f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 4 May 2018 23:38:42 -0700 Subject: [PATCH 1/3] add design --- weed/filer/vasto_store/design.txt | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 weed/filer/vasto_store/design.txt diff --git a/weed/filer/vasto_store/design.txt b/weed/filer/vasto_store/design.txt new file mode 100644 index 000000000..99faa29d6 --- /dev/null +++ b/weed/filer/vasto_store/design.txt @@ -0,0 +1,45 @@ +There are two main components of a filer: directories and files. + +My previous approach was to use some sequance number to generate directoryId. +However, this is not scalable. The id generation itself is a bottleneck. +It needs careful locking and deduplication checking to get a directoryId. + +In a second design, each directory is deterministically mapped to UUID version 3, +which uses MD5 to map a tuple of to a version 3 UUID. +However, this UUID3 approach is logically the same as storing the full path. + +Storing the full path is the simplest design. + +separator is a special byte, 0x00. + +When writing a file: + => fildId, file properties +For folders: + The filer breaks the directory path into folders. + for each folder: + if it is not in cache: + check whether the folder is created in the KVS, if not: + set => directory properties + if no permission for the folder: + break + + +The filer caches the most recently used folder permissions with a TTL. + So any folder permission change needs to wait TTL interval to take effect. + + + +When listing the directory: + prefix scan of using (the folder full path + separator) as the prefix + +The downside: + 1. Rename a folder will need to recursively process all sub folders and files. + 2. Move a folder will need to recursively process all sub folders and files. +So these operations are not allowed if the folder is not empty. + +Allowing: + 1. Rename a file + 2. Move a file to a different folder + 3. Delete an empty folder + + From fffb14bc87f930cfdc14ab6fba01218606463f34 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 5 May 2018 02:01:50 -0700 Subject: [PATCH 2/3] better support FUSE Lookup() --- weed/command/mount_std.go | 27 +++++++-------- weed/filer/client_operations.go | 19 ++++++++++- weed/filer/embedded_filer/directory.go | 8 ++--- weed/filer/embedded_filer/directory_in_map.go | 22 ++++++------ weed/filer/embedded_filer/directory_test.go | 8 ++--- weed/filer/embedded_filer/filer_embedded.go | 34 ++++++++++++------- weed/filer/embedded_filer/files_in_leveldb.go | 10 +++--- weed/filer/filer.go | 11 ++---- .../flat_namespace/flat_namespace_filer.go | 10 ++++-- weed/filer/postgres_store/postgres_native.go | 14 ++++---- weed/filer/postgres_store/postgres_store.go | 21 ++++++++---- weed/server/filer_server_handlers_api.go | 9 ++++- 12 files changed, 117 insertions(+), 76 deletions(-) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 0c36e488f..ef962e15b 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -59,27 +59,27 @@ func (WFS) Root() (fs.Node, error) { var fileIdMap = make(map[uint64]filer.FileId) type Dir struct { - Id uint64 Path string DirentMap map[string]*fuse.Dirent } func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error { - attr.Inode = dir.Id attr.Mode = os.ModeDir | 0555 return nil } func (dir *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) { - if dirent, ok := dir.DirentMap[name]; ok { - if dirent.Type == fuse.DT_File { - return &File{Id: dirent.Inode, FileId: fileIdMap[dirent.Inode], Name: dirent.Name}, nil + if entry, err := filer.LookupDirectoryEntry(*mountOptions.filer, dir.Path, name); err == nil { + if !entry.Found { + return nil, fuse.ENOENT + } + if entry.FileId != "" { + return &File{FileId: filer.FileId(entry.FileId), Name: name}, nil + } else { + return &Dir{Path: path.Join(dir.Path, name)}, nil } - return &Dir{ - Id: dirent.Inode, - Path: path.Join(dir.Path, dirent.Name), - }, nil } + return nil, fuse.ENOENT } @@ -90,17 +90,16 @@ func (dir *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { } if dirs, e := filer.ListDirectories(*mountOptions.filer, dir.Path); e == nil { for _, d := range dirs.Directories { - dirId := uint64(d.Id) - dirent := fuse.Dirent{Inode: dirId, Name: d.Name, Type: fuse.DT_Dir} + dirent := fuse.Dirent{Name: string(d), Type: fuse.DT_Dir} ret = append(ret, dirent) - dir.DirentMap[d.Name] = &dirent + dir.DirentMap[string(d)] = &dirent } } if files, e := filer.ListFiles(*mountOptions.filer, dir.Path, ""); e == nil { for _, f := range files.Files { if fileId, e := storage.ParseFileId(string(f.Id)); e == nil { fileInode := uint64(fileId.VolumeId)<<48 + fileId.Key - dirent := fuse.Dirent{Inode: fileInode, Name: f.Name, Type: fuse.DT_File} + dirent := fuse.Dirent{Name: f.Name, Type: fuse.DT_File} ret = append(ret, dirent) dir.DirentMap[f.Name] = &dirent fileIdMap[fileInode] = f.Id @@ -120,13 +119,11 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { } type File struct { - Id uint64 FileId filer.FileId Name string } func (file *File) Attr(context context.Context, attr *fuse.Attr) error { - attr.Inode = file.Id attr.Mode = 0444 ret, err := filer.GetFileSize(*mountOptions.filer, string(file.FileId)) if err == nil { diff --git a/weed/filer/client_operations.go b/weed/filer/client_operations.go index 95dd26396..057fb4c66 100644 --- a/weed/filer/client_operations.go +++ b/weed/filer/client_operations.go @@ -65,7 +65,7 @@ func GetFileContent(server string, fileId string) (ret *GetFileContentResult, er } type ListDirectoriesResult struct { - Directories []DirectoryEntry + Directories []DirectoryName Error string `json:"error,omitempty"` } @@ -80,6 +80,23 @@ func ListDirectories(server string, directory string) (ret *ListDirectoriesResul return nil, err } +type LookupDirectoryEntryResult struct { + Found bool + FileId string + Error string `json:"error,omitempty"` +} + +func LookupDirectoryEntry(server string, directory string, name string) (ret *LookupDirectoryEntryResult, err error) { + ret = new(LookupDirectoryEntryResult) + if err := call(server, ApiRequest{Command: "lookupDirectoryEntry", Directory: directory, FileName: name}, ret); err == nil { + if ret.Error != "" { + return nil, errors.New(ret.Error) + } + return ret, nil + } + return nil, err +} + func DeleteDirectoryOrFile(server string, path string, isDir bool) error { destUrl := fmt.Sprintf("http://%s%s", server, path) if isDir { diff --git a/weed/filer/embedded_filer/directory.go b/weed/filer/embedded_filer/directory.go index 4d4bd1c59..1ff725f28 100644 --- a/weed/filer/embedded_filer/directory.go +++ b/weed/filer/embedded_filer/directory.go @@ -5,11 +5,11 @@ import ( ) type DirectoryManager interface { - FindDirectory(dirPath string) (filer.DirectoryId, error) - ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) - MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error) + FindDirectory(dirPath string) (DirectoryId, error) + ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) + MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error) MoveUnderDirectory(oldDirPath string, newParentDirPath string) error DeleteDirectory(dirPath string) error //functions used by FUSE - FindDirectoryById(filer.DirectoryId, error) + FindDirectoryById(DirectoryId, error) } diff --git a/weed/filer/embedded_filer/directory_in_map.go b/weed/filer/embedded_filer/directory_in_map.go index e082bbf81..1266d3779 100644 --- a/weed/filer/embedded_filer/directory_in_map.go +++ b/weed/filer/embedded_filer/directory_in_map.go @@ -16,12 +16,14 @@ import ( var writeLock sync.Mutex //serialize changes to dir.log +type DirectoryId int32 + type DirectoryEntryInMap struct { sync.Mutex Name string Parent *DirectoryEntryInMap subDirectories map[string]*DirectoryEntryInMap - Id filer.DirectoryId + Id DirectoryId } func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) { @@ -45,18 +47,18 @@ func (de *DirectoryEntryInMap) hasChildren() bool { defer de.Unlock() return len(de.subDirectories) > 0 } -func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) { +func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryName) { de.Lock() defer de.Unlock() - for k, v := range de.subDirectories { - dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id}) + for k, _ := range de.subDirectories { + dirNames = append(dirNames, filer.DirectoryName(k)) } return dirNames } type DirectoryManagerInMap struct { Root *DirectoryEntryInMap - max filer.DirectoryId + max DirectoryId logFile *os.File isLoading bool } @@ -113,7 +115,7 @@ func (dm *DirectoryManagerInMap) processEachLine(line string) error { if pe != nil { return pe } - if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil { + if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil { return e } case "mov": @@ -172,7 +174,7 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI } return dir, nil } -func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) { +func (dm *DirectoryManagerInMap) findDirectoryId(dirPath string) (DirectoryId, error) { d, e := dm.findDirectory(dirPath) if e == nil { return d.Id, nil @@ -180,7 +182,7 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryI return dm.Root.Id, e } -func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error { +func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error { dirPath = CleanFilePath(dirPath) if dirPath == "/" { return nil @@ -248,7 +250,7 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn return dir, created } -func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) { +func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) { dir, _ := dm.makeDirectory(dirPath) return dir.Id, nil } @@ -275,7 +277,7 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent return nil } -func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) { +func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryName, err error) { d, e := dm.findDirectory(dirPath) if e != nil { return dirNames, e diff --git a/weed/filer/embedded_filer/directory_test.go b/weed/filer/embedded_filer/directory_test.go index c8b3f1f30..b70164a14 100644 --- a/weed/filer/embedded_filer/directory_test.go +++ b/weed/filer/embedded_filer/directory_test.go @@ -19,17 +19,17 @@ func TestDirectory(t *testing.T) { dm.MakeDirectory("/a/b/e/f") dm.MakeDirectory("/a/b/e/f/g") dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t") - if _, err := dm.FindDirectory("/a/b/e/f/g"); err == nil { + if _, err := dm.findDirectoryId("/a/b/e/f/g"); err == nil { t.Fatal("/a/b/e/f/g should not exist any more after moving") } - if _, err := dm.FindDirectory("/a/b/t"); err != nil { + if _, err := dm.findDirectoryId("/a/b/t"); err != nil { t.Fatal("/a/b/t should exist after moving") } - if _, err := dm.FindDirectory("/a/b/g"); err == nil { + if _, err := dm.findDirectoryId("/a/b/g"); err == nil { t.Fatal("/a/b/g should not exist after moving") } dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "") - if _, err := dm.FindDirectory("/a/b/f"); err != nil { + if _, err := dm.findDirectoryId("/a/b/f"); err != nil { t.Fatal("/a/b/g should not exist after moving") } dm.MakeDirectory("/a/b/g/h/i") diff --git a/weed/filer/embedded_filer/filer_embedded.go b/weed/filer/embedded_filer/filer_embedded.go index 4a335058b..d6ab3c70a 100644 --- a/weed/filer/embedded_filer/filer_embedded.go +++ b/weed/filer/embedded_filer/filer_embedded.go @@ -45,27 +45,37 @@ func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error) } func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) { dir, file := filepath.Split(filePath) - dirId, e := filer.directories.FindDirectory(dir) + return filer.findFileEntry(dir, file) +} +func (filer *FilerEmbedded) findFileEntry(parentPath string, fileName string) (fid string, err error) { + dirId, e := filer.directories.findDirectoryId(parentPath) if e != nil { return "", e } - return filer.files.FindFile(dirId, file) + return filer.files.FindFile(dirId, fileName) } -func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) { - return filer.directories.FindDirectory(dirPath) + +func (filer *FilerEmbedded) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) { + if _, err = filer.directories.findDirectory(filepath.Join(dirPath, name)); err == nil { + return true, "", nil + } + if fileId, err = filer.findFileEntry(dirPath, name); err == nil { + return true, fileId, nil + } + return false, "", err } -func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) { +func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) { return filer.directories.ListDirectories(dirPath) } func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { - dirId, e := filer.directories.FindDirectory(dirPath) + dirId, e := filer.directories.findDirectoryId(dirPath) if e != nil { return nil, e } return filer.files.ListFiles(dirId, lastFileName, limit), nil } func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) { - dirId, e := filer.directories.FindDirectory(dirPath) + dirId, e := filer.directories.findDirectoryId(dirPath) if e != nil { return e } @@ -74,7 +84,7 @@ func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs)) } for _, sub := range sub_dirs { - if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, sub.Name), recursive); delete_sub_err != nil { + if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, string(sub)), recursive); delete_sub_err != nil { return delete_sub_err } } @@ -108,7 +118,7 @@ func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) { dir, file := filepath.Split(filePath) - dirId, e := filer.directories.FindDirectory(dir) + dirId, e := filer.directories.findDirectoryId(dir) if e != nil { return "", e } @@ -126,8 +136,8 @@ func (filer *FilerEmbedded) Move(fromPath string, toPath string) error { filer.mvMutex.Lock() defer filer.mvMutex.Unlock() - if _, dir_err := filer.FindDirectory(fromPath); dir_err == nil { - if _, err := filer.FindDirectory(toPath); err == nil { + if _, dir_err := filer.directories.findDirectoryId(fromPath); dir_err == nil { + if _, err := filer.directories.findDirectoryId(toPath); err == nil { // move folder under an existing folder return filer.directories.MoveUnderDirectory(fromPath, toPath, "") } @@ -135,7 +145,7 @@ func (filer *FilerEmbedded) Move(fromPath string, toPath string) error { return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath)) } if fid, file_err := filer.DeleteFile(fromPath); file_err == nil { - if _, err := filer.FindDirectory(toPath); err == nil { + if _, err := filer.directories.findDirectoryId(toPath); err == nil { // move file under an existing folder return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid) } diff --git a/weed/filer/embedded_filer/files_in_leveldb.go b/weed/filer/embedded_filer/files_in_leveldb.go index c40d7adaf..bde41d8a2 100644 --- a/weed/filer/embedded_filer/files_in_leveldb.go +++ b/weed/filer/embedded_filer/files_in_leveldb.go @@ -28,7 +28,7 @@ func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) { return } -func genKey(dirId filer.DirectoryId, fileName string) []byte { +func genKey(dirId DirectoryId, fileName string) []byte { ret := make([]byte, 0, 4+len(fileName)) for i := 3; i >= 0; i-- { ret = append(ret, byte(dirId>>(uint(i)*8))) @@ -37,11 +37,11 @@ func genKey(dirId filer.DirectoryId, fileName string) []byte { return ret } -func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) { +func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) { glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid) return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil) } -func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) { +func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) { if fid, err = fl.FindFile(dirId, fileName); err != nil { if err == leveldb.ErrNotFound { return "", nil @@ -51,7 +51,7 @@ func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string err = fl.db.Delete(genKey(dirId, fileName), nil) return fid, err } -func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) { +func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) { data, e := fl.db.Get(genKey(dirId, fileName), nil) if e == leveldb.ErrNotFound { return "", filer.ErrNotFound @@ -60,7 +60,7 @@ func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) } return string(data), nil } -func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) { +func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) { glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit) dirKey := genKey(dirId, "") iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 5d5acb68d..8f6344975 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -11,12 +11,7 @@ type FileEntry struct { Id FileId `json:"fid,omitempty"` } -type DirectoryId int32 - -type DirectoryEntry struct { - Name string //dir name without path - Id DirectoryId -} +type DirectoryName string type Filer interface { CreateFile(fullFileName string, fid string) (err error) @@ -24,11 +19,11 @@ type Filer interface { DeleteFile(fullFileName string) (fid string, err error) //Optional functions. embedded filer support these - FindDirectory(dirPath string) (dirId DirectoryId, err error) - ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) + ListDirectories(dirPath string) (dirs []DirectoryName, err error) ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) DeleteDirectory(dirPath string, recursive bool) (err error) Move(fromPath string, toPath string) (err error) + LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) } var ErrNotFound = errors.New("filer: no entry is found in filer store") diff --git a/weed/filer/flat_namespace/flat_namespace_filer.go b/weed/filer/flat_namespace/flat_namespace_filer.go index f5e17b621..4173c41ee 100644 --- a/weed/filer/flat_namespace/flat_namespace_filer.go +++ b/weed/filer/flat_namespace/flat_namespace_filer.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/chrislusf/seaweedfs/weed/filer" + "path/filepath" ) type FlatNamespaceFiler struct { @@ -28,10 +29,13 @@ func (filer *FlatNamespaceFiler) CreateFile(fullFileName string, fid string) (er func (filer *FlatNamespaceFiler) FindFile(fullFileName string) (fid string, err error) { return filer.store.Get(fullFileName) } -func (filer *FlatNamespaceFiler) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) { - return 0, ErrNotImplemented +func (filer *FlatNamespaceFiler) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) { + if fileId, err = filer.FindFile(filepath.Join(dirPath, name)); err == nil { + return true, fileId, nil + } + return false, "", err } -func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) { +func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) { return nil, ErrNotImplemented } func (filer *FlatNamespaceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { diff --git a/weed/filer/postgres_store/postgres_native.go b/weed/filer/postgres_store/postgres_native.go index 12b5015cf..61bd4210c 100644 --- a/weed/filer/postgres_store/postgres_native.go +++ b/weed/filer/postgres_store/postgres_native.go @@ -14,6 +14,8 @@ import ( "strings" ) +type DirectoryId int32 + func databaseExists(db *sql.DB, databaseName string) (bool, error) { sqlStatement := "SELECT datname from pg_database WHERE datname='%s'" row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName)) @@ -293,13 +295,13 @@ func (s *PostgresStore) delete(uriPath string) error { return nil } -func (s *PostgresStore) lookupDirectory(dirPath string) (filer.DirectoryId, string, error) { +func (s *PostgresStore) lookupDirectory(dirPath string) (DirectoryId, string, error) { directoryRoot, directoryName := s.mySplitPath(dirPath) sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName) row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName) - var id filer.DirectoryId + var id DirectoryId var dirRoot string var dirName string err := row.Scan(&id, &dirRoot, &dirName) @@ -312,7 +314,7 @@ func (s *PostgresStore) lookupDirectory(dirPath string) (filer.DirectoryId, stri return id, filepath.Join(dirRoot, dirName), err } -func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryEntry, err error) { +func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryName, err error) { sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName) rows, err := s.db.Query(sqlStatement, dirPath, limit) @@ -323,7 +325,7 @@ func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer if rows != nil { defer rows.Close() for rows.Next() { - var id filer.DirectoryId + var id DirectoryId var directoryRoot string var directoryName string @@ -331,7 +333,7 @@ func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer if scanErr != nil { err = scanErr } - dirs = append(dirs, filer.DirectoryEntry{Name: (directoryName), Id: id}) + dirs = append(dirs, filer.DirectoryName(directoryName)) } } return @@ -344,7 +346,7 @@ func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bo sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName) row := s.db.QueryRow(sqlStatement, dirPath+"%") - var id filer.DirectoryId + var id DirectoryId err := row.Scan(&id) if err != nil { if err == sql.ErrNoRows { diff --git a/weed/filer/postgres_store/postgres_store.go b/weed/filer/postgres_store/postgres_store.go index e0aa2a02f..854793f6a 100644 --- a/weed/filer/postgres_store/postgres_store.go +++ b/weed/filer/postgres_store/postgres_store.go @@ -10,6 +10,7 @@ import ( _ "github.com/lib/pq" _ "path/filepath" + "path/filepath" ) const ( @@ -74,6 +75,17 @@ func (s *PostgresStore) FindFile(fullFilePath string) (fid string, err error) { return fid, err } +func (s *PostgresStore) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) { + fullPath := filepath.Join(dirPath, name) + if fileId, err = s.FindFile(fullPath); err == nil { + return true, fileId, nil + } + if _, _, err := s.lookupDirectory(fullPath); err == nil { + return true, "", err + } + return false, "", err +} + func (s *PostgresStore) DeleteFile(fullFilePath string) (fid string, err error) { if err != nil { return "", fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err) @@ -90,14 +102,9 @@ func (s *PostgresStore) DeleteFile(fullFilePath string) (fid string, err error) } } -func (s *PostgresStore) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) { - dirId, _, err = s.lookupDirectory(dirPath) - return dirId, err -} +func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) { -func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) { - - dirs, err = s.findDirectories(dirPath, 20) + dirs, err = s.findDirectories(dirPath, 1000) glog.V(3).Infof("Postgres ListDirs = found %d directories under %s", len(dirs), dirPath) diff --git a/weed/server/filer_server_handlers_api.go b/weed/server/filer_server_handlers_api.go index f07aab88a..93a0dd751 100644 --- a/weed/server/filer_server_handlers_api.go +++ b/weed/server/filer_server_handlers_api.go @@ -21,6 +21,13 @@ func (fs *FilerServer) apiHandler(w http.ResponseWriter, r *http.Request) { writeJsonError(w, r, http.StatusInternalServerError, err) } switch apiRequest.Command { + case "lookupDirectoryEntry": + res := filer.LookupDirectoryEntryResult{} + res.Found, res.FileId, err = fs.filer.LookupDirectoryEntry(apiRequest.Directory, apiRequest.FileName) + if err != nil { + res.Error = err.Error() + } + writeJsonQuiet(w, r, http.StatusOK, res) case "listDirectories": res := filer.ListDirectoriesResult{} res.Directories, err = fs.filer.ListDirectories(apiRequest.Directory) @@ -30,7 +37,7 @@ func (fs *FilerServer) apiHandler(w http.ResponseWriter, r *http.Request) { writeJsonQuiet(w, r, http.StatusOK, res) case "listFiles": res := filer.ListFilesResult{} - res.Files, err = fs.filer.ListFiles(apiRequest.Directory, apiRequest.FileName, 100) + res.Files, err = fs.filer.ListFiles(apiRequest.Directory, apiRequest.FileName, 1000) if err != nil { res.Error = err.Error() } From 7f49514f6fad19b70d955f8ad94fa7eae4e45edf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 5 May 2018 14:10:50 -0700 Subject: [PATCH 3/3] fix filer UI --- weed/server/filer_ui/templates.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index 6f4f7ce86..b6a8a1878 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -28,15 +28,17 @@ var StatusTpl = template.Must(template.New("status").Parse(` {{$path := .Path }} {{ range $dirs_index, $dir := .Directories }}
  • - - {{ $dir.Name }} + + {{ $dir }}
    • {{ end }} {{ range $file_index, $file := .Files }}
  • + {{ $file.Name }} +
    • {{ end }}