From 58954bf46f5098c1181920bd04ac9e45f6b97913 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 14 May 2018 02:02:17 -0700 Subject: [PATCH] pass file attributes from filer to mount --- weed/filer2/filer_structure.go | 9 ++++++- weed/filesys/dir.go | 4 ++- weed/filesys/file.go | 46 +++++++++++++++++++++++--------- weed/filesys/wfs.go | 7 +++-- weed/server/filer_grpc_server.go | 28 ++++++++++++------- 5 files changed, 68 insertions(+), 26 deletions(-) diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 1cff96253..b2c1f5902 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -10,6 +10,13 @@ import ( type FileId string //file id in SeaweedFS type FullPath string +func NewFullPath(dir, name string) FullPath { + if dir == "/" { + return FullPath(dir + name) + } + return FullPath(dir + "/" + name) +} + func (fp FullPath) DirAndName() (string, string) { dir, name := filepath.Split(string(fp)) if dir == "/" { @@ -35,7 +42,7 @@ type Attr struct { } func (attr Attr) IsDirectory() (bool) { - return attr.Mode & os.ModeDir > 0 + return attr.Mode&os.ModeDir > 0 } type Entry struct { diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index c1c6afe9c..d408dc1ac 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -12,6 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "time" ) type Dir struct { @@ -74,7 +75,7 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro if entry.IsDirectory { node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs} } else { - node = &File{FileId: filer.FileId(entry.FileId), Name: name, wfs: dir.wfs} + node = &File{FileId: filer.FileId(entry.FileId), Name: name, dir: dir, wfs: dir.wfs} } dir.NodeMap[name] = node return node, nil @@ -104,6 +105,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { } else { dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File} ret = append(ret, dirent) + dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry.Attributes, 3*time.Second) } } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index e4c79c055..b4a447fa6 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -9,6 +9,9 @@ import ( "bazil.org/fuse/fs" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "path/filepath" + "os" + "time" ) var _ = fs.Node(&File{}) @@ -22,29 +25,48 @@ var _ = fs.HandleWriter(&File{}) type File struct { FileId filer.FileId Name string + dir *Dir wfs *WFS } func (file *File) Attr(context context.Context, attr *fuse.Attr) error { - attr.Mode = 0444 - return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + fullPath := filepath.Join(file.dir.Path, file.Name) + item := file.wfs.listDirectoryEntriesCache.Get(fullPath) + var attributes *filer_pb.FuseAttributes + if item != nil { + attributes = item.Value().(*filer_pb.FuseAttributes) + glog.V(1).Infof("read cached file %v attributes", file.Name) + } else { + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - request := &filer_pb.GetFileAttributesRequest{ - Name: file.Name, - ParentDir: "", //TODO add parent folder - FileId: string(file.FileId), - } + request := &filer_pb.GetFileAttributesRequest{ + Name: file.Name, + ParentDir: file.dir.Path, + } + + glog.V(1).Infof("read file size: %v", request) + resp, err := client.GetFileAttributes(context, request) + if err != nil { + return err + } + + attributes = resp.Attributes + + return nil + }) - glog.V(1).Infof("read file size: %v", request) - resp, err := client.GetFileAttributes(context, request) if err != nil { return err } + } - attr.Size = resp.Attributes.FileSize + attr.Mode = os.FileMode(attributes.FileMode) + attr.Size = attributes.FileSize + attr.Mtime = time.Unix(attributes.Mtime, 0) + attr.Gid = attributes.Gid + attr.Uid = attributes.Uid + return nil - return nil - }) } func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index da50ae4a4..9f8f4649a 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -5,15 +5,18 @@ import ( "fmt" "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/karlseguin/ccache" ) type WFS struct { - filer string + filer string + listDirectoryEntriesCache *ccache.Cache } func NewSeaweedFileSystem(filer string) *WFS { return &WFS{ - filer: filer, + filer: filer, + listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)), } } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index e5b30aedf..4d31782da 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -2,8 +2,6 @@ package weed_server import ( "context" - "strconv" - "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -50,10 +48,19 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie if !entry.IsDirectory() && len(entry.Chunks) > 0 { fileId = string(entry.Chunks[0].Fid) } + + glog.V(0).Infof("%s attr=%v size=%d", entry.Name(), entry.Attr, filer2.Chunks(entry.Chunks).TotalSize()) resp.Entries = append(resp.Entries, &filer_pb.Entry{ Name: entry.Name(), IsDirectory: entry.IsDirectory(), FileId: fileId, + Attributes: &filer_pb.FuseAttributes{ + FileSize: filer2.Chunks(entry.Chunks).TotalSize(), + Mtime: entry.Mtime.Unix(), + Gid: entry.Gid, + Uid: entry.Uid, + FileMode: uint32(entry.Mode), + }, }) } @@ -64,17 +71,18 @@ func (fs *FilerServer) GetFileAttributes(ctx context.Context, req *filer_pb.GetF attributes := &filer_pb.FuseAttributes{} - server, err := operation.LookupFileId(fs.getMasterNode(), req.FileId) + found, entry, err := fs.filer.FindEntry(filer2.NewFullPath(req.ParentDir, req.Name)) if err != nil { return nil, err } - head, err := util.Head(server) - if err != nil { - return nil, err - } - attributes.FileSize, err = strconv.ParseUint(head.Get("Content-Length"), 10, 0) - if err != nil { - return nil, err + if !found { + attributes.FileSize = 0 + } else { + attributes.FileSize = filer2.Chunks(entry.Chunks).TotalSize() + attributes.FileMode = uint32(entry.Mode) + attributes.Uid = entry.Uid + attributes.Gid = entry.Gid + attributes.Mtime = entry.Mtime.Unix() } return &filer_pb.GetFileAttributesResponse{