gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

delay file creation without chunks from dir.Create to file.Flush

Browse source
This commit is contained in:
Chris Lu 2018-09-22 00:11:46 -07:00
parent f905a1e779
commit 7bb62b9f21
6 changed files with 99 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
weed/command/filer_replication.go
View file

@ -52,7 +52,13 @@ func runFilerReplicate(cmd *Command, args []string) bool {
glog.Errorf("receive %s: %+v", key, err) glog.Errorf("receive %s: %+v", key, err)
continue continue
} }
glog.V(1).Infof("processing file: %s", key) if m.OldEntry!=nil&&m.NewEntry==nil{
glog.V(1).Infof("delete: %s", key)
}else if m.OldEntry==nil&&m.NewEntry!=nil{
glog.V(1).Infof(" add: %s", key)
}else{
glog.V(1).Infof("modify: %s", key)
}
if err = replicator.Replicate(key, m); err != nil { if err = replicator.Replicate(key, m); err != nil {
glog.Errorf("replicate %s: %+v", key, err) glog.Errorf("replicate %s: %+v", key, err)
} }

10
weed/filer2/filer.go
View file

@ -121,8 +121,14 @@ func (f *Filer) CreateEntry(entry *Entry) error {
oldEntry, _ := f.FindEntry(entry.FullPath) oldEntry, _ := f.FindEntry(entry.FullPath)
if err := f.store.InsertEntry(entry); err != nil { if oldEntry == nil {
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err) if err := f.store.InsertEntry(entry); err != nil {
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
}
} else {
if err := f.store.UpdateEntry(entry); err != nil {
return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
}
} }
f.NotifyUpdateEvent(oldEntry, entry, true) f.NotifyUpdateEvent(oldEntry, entry, true)

80
weed/filesys/dir.go
View file

@ -91,55 +91,57 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
return nil return nil
} }
func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk, attr *filer_pb.FuseAttributes) *File { func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
return &File{ return &File{
Name: name, Name: name,
dir: dir, dir: dir,
wfs: dir.wfs, wfs: dir.wfs,
attributes: attr, entry: entry,
Chunks: chunks,
} }
} }
func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.CreateEntryRequest{
Directory: dir.Path,
request := &filer_pb.CreateEntryRequest{ Entry: &filer_pb.Entry{
Directory: dir.Path, Name: req.Name,
Entry: &filer_pb.Entry{ IsDirectory: req.Mode&os.ModeDir > 0,
Name: req.Name, Attributes: &filer_pb.FuseAttributes{
IsDirectory: req.Mode&os.ModeDir > 0, Mtime: time.Now().Unix(),
Attributes: &filer_pb.FuseAttributes{ Crtime: time.Now().Unix(),
Mtime: time.Now().Unix(), FileMode: uint32(req.Mode),
Crtime: time.Now().Unix(), Uid: req.Uid,
FileMode: uint32(req.Mode), Gid: req.Gid,
Uid: req.Uid, Collection: dir.wfs.option.Collection,
Gid: req.Gid, Replication: dir.wfs.option.Replication,
Collection: dir.wfs.option.Collection, TtlSec: dir.wfs.option.TtlSec,
Replication: dir.wfs.option.Replication,
TtlSec: dir.wfs.option.TtlSec,
},
}, },
},
}
glog.V(1).Infof("create: %v", request)
if request.Entry.IsDirectory {
if err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
if _, err := client.CreateEntry(ctx, request); err != nil {
glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
return fuse.EIO
}
return nil
}); err != nil {
return nil, nil, err
} }
glog.V(1).Infof("create: %v", request)
if _, err := client.CreateEntry(ctx, request); err != nil {
glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
return fuse.EIO
}
return nil
})
if err == nil {
file := dir.newFile(req.Name, nil, &filer_pb.FuseAttributes{})
file.isOpen = true
return file, dir.wfs.AcquireHandle(file, req.Uid, req.Gid), nil
} }
return nil, nil, err file := dir.newFile(req.Name, request.Entry)
if !request.Entry.IsDirectory {
file.isOpen = true
}
fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
fh.dirtyMetadata = true
return file, fh, nil
} }
func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
@ -204,7 +206,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
if entry.IsDirectory { if entry.IsDirectory {
node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes} node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes}
} else { } else {
node = dir.newFile(req.Name, entry.Chunks, entry.Attributes) node = dir.newFile(req.Name, entry)
} }
resp.EntryValid = time.Duration(0) resp.EntryValid = time.Duration(0)

3
weed/filesys/dirty_page.go
View file

@ -66,7 +66,8 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da
if offset != pages.Offset+pages.Size { if offset != pages.Offset+pages.Size {
// when this happens, debug shows the data overlapping with existing data is empty // when this happens, debug shows the data overlapping with existing data is empty
// the data is not just append // the data is not just append
if offset == pages.Offset { if offset == pages.Offset && int(pages.Size) < len(data) {
// glog.V(2).Infof("pages[%d,%d) pages.Data len=%v, data len=%d, pages.Size=%d", pages.Offset, pages.Offset+pages.Size, len(pages.Data), len(data), pages.Size)
copy(pages.Data[pages.Size:], data[pages.Size:]) copy(pages.Data[pages.Size:], data[pages.Size:])
} else { } else {
if pages.Size != 0 { if pages.Size != 0 {

57
weed/filesys/file.go
View file

@ -18,12 +18,11 @@ var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{}) var _ = fs.NodeSetattrer(&File{})
type File struct { type File struct {
Chunks []*filer_pb.FileChunk Name string
Name string dir *Dir
dir *Dir wfs *WFS
wfs *WFS entry *filer_pb.Entry
attributes *filer_pb.FuseAttributes isOpen bool
isOpen bool
} }
func (file *File) fullpath() string { func (file *File) fullpath() string {
@ -36,11 +35,11 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
return err return err
} }
attr.Mode = os.FileMode(file.attributes.FileMode) attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
attr.Size = filer2.TotalSize(file.Chunks) attr.Size = filer2.TotalSize(file.entry.Chunks)
attr.Mtime = time.Unix(file.attributes.Mtime, 0) attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0)
attr.Gid = file.attributes.Gid attr.Gid = file.entry.Attributes.Gid
attr.Uid = file.attributes.Uid attr.Uid = file.entry.Attributes.Uid
return nil return nil
@ -72,41 +71,37 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
return nil return nil
} }
glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.attributes) glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
if req.Valid.Size() { if req.Valid.Size() {
glog.V(3).Infof("%v file setattr set size=%v", file.fullpath(), req.Size) glog.V(3).Infof("%v file setattr set size=%v", file.fullpath(), req.Size)
if req.Size == 0 { if req.Size == 0 {
// fmt.Printf("truncate %v \n", fullPath) // fmt.Printf("truncate %v \n", fullPath)
file.Chunks = nil file.entry.Chunks = nil
} }
file.attributes.FileSize = req.Size file.entry.Attributes.FileSize = req.Size
} }
if req.Valid.Mode() { if req.Valid.Mode() {
file.attributes.FileMode = uint32(req.Mode) file.entry.Attributes.FileMode = uint32(req.Mode)
} }
if req.Valid.Uid() { if req.Valid.Uid() {
file.attributes.Uid = req.Uid file.entry.Attributes.Uid = req.Uid
} }
if req.Valid.Gid() { if req.Valid.Gid() {
file.attributes.Gid = req.Gid file.entry.Attributes.Gid = req.Gid
} }
if req.Valid.Mtime() { if req.Valid.Mtime() {
file.attributes.Mtime = req.Mtime.Unix() file.entry.Attributes.Mtime = req.Mtime.Unix()
} }
return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.UpdateEntryRequest{ request := &filer_pb.UpdateEntryRequest{
Directory: file.dir.Path, Directory: file.dir.Path,
Entry: &filer_pb.Entry{ Entry: file.entry,
Name: file.Name,
Attributes: file.attributes,
Chunks: file.Chunks,
},
} }
glog.V(1).Infof("set attr file entry: %v", request) glog.V(1).Infof("set attr file entry: %v", request)
@ -130,31 +125,29 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
} }
func (file *File) maybeLoadAttributes(ctx context.Context) error { func (file *File) maybeLoadAttributes(ctx context.Context) error {
if file.attributes == nil || !file.isOpen { if file.entry == nil || !file.isOpen {
item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath()) item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
if item != nil && !item.Expired() { if item != nil && !item.Expired() {
entry := item.Value().(*filer_pb.Entry) entry := item.Value().(*filer_pb.Entry)
file.Chunks = entry.Chunks file.entry = entry
file.attributes = entry.Attributes
// glog.V(1).Infof("file attr read cached %v attributes", file.Name) // glog.V(1).Infof("file attr read cached %v attributes", file.Name)
} else { } else {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.GetEntryAttributesRequest{ request := &filer_pb.LookupDirectoryEntryRequest{
Name: file.Name, Name: file.Name,
ParentDir: file.dir.Path, Directory: file.dir.Path,
} }
resp, err := client.GetEntryAttributes(ctx, request) resp, err := client.LookupDirectoryEntry(ctx, request)
if err != nil { if err != nil {
glog.V(0).Infof("file attr read file %v: %v", request, err) glog.V(0).Infof("file attr read file %v: %v", request, err)
return err return err
} }
file.attributes = resp.Attributes file.entry = resp.Entry
file.Chunks = resp.Chunks
glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.attributes, filer2.TotalSize(file.Chunks)) glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))
return nil return nil
}) })

49
weed/filesys/filehandle.go
View file

@ -17,10 +17,9 @@ import (
type FileHandle struct { type FileHandle struct {
// cache file has been written to // cache file has been written to
dirtyPages *ContinuousDirtyPages dirtyPages *ContinuousDirtyPages
dirtyMetadata bool
contentType string contentType string
dirtyMetadata bool
handle uint64 handle uint64
f *File f *File
RequestId fuse.RequestID // unique ID for request RequestId fuse.RequestID // unique ID for request
@ -51,14 +50,14 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
glog.V(4).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size)) glog.V(4).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size))
// this value should come from the filer instead of the old f // this value should come from the filer instead of the old f
if len(fh.f.Chunks) == 0 { if len(fh.f.entry.Chunks) == 0 {
glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name) glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name)
return fmt.Errorf("empty file %v/%v", fh.f.dir.Path, fh.f.Name) return fmt.Errorf("empty file %v/%v", fh.f.dir.Path, fh.f.Name)
} }
buff := make([]byte, req.Size) buff := make([]byte, req.Size)
chunkViews := filer2.ViewFromChunks(fh.f.Chunks, req.Offset, req.Size) chunkViews := filer2.ViewFromChunks(fh.f.entry.Chunks, req.Offset, req.Size)
var vids []string var vids []string
for _, chunkView := range chunkViews { for _, chunkView := range chunkViews {
@ -152,7 +151,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
} }
for _, chunk := range chunks { for _, chunk := range chunks {
fh.f.Chunks = append(fh.f.Chunks, chunk) fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
fh.dirtyMetadata = true fh.dirtyMetadata = true
} }
@ -184,52 +183,36 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
return fmt.Errorf("flush %s/%s to %s [%d,%d): %v", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size), err) return fmt.Errorf("flush %s/%s to %s [%d,%d): %v", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
} }
if chunk != nil { if chunk != nil {
fh.f.Chunks = append(fh.f.Chunks, chunk) fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
fh.dirtyMetadata = true
} }
if !fh.dirtyMetadata { if !fh.dirtyMetadata {
return nil return nil
} }
if len(fh.f.Chunks) == 0 { return fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
glog.V(2).Infof("fh %s/%s flush skipping empty: %v", fh.f.dir.Path, fh.f.Name, req)
return nil
}
err = fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { if fh.f.entry.Attributes != nil {
fh.f.entry.Attributes.Mime = fh.contentType
if fh.f.attributes != nil { fh.f.entry.Attributes.Uid = req.Uid
fh.f.attributes.Mime = fh.contentType fh.f.entry.Attributes.Gid = req.Gid
fh.f.attributes.Uid = req.Uid
fh.f.attributes.Gid = req.Gid
} }
request := &filer_pb.UpdateEntryRequest{ request := &filer_pb.CreateEntryRequest{
Directory: fh.f.dir.Path, Directory: fh.f.dir.Path,
Entry: &filer_pb.Entry{ Entry: fh.f.entry,
Name: fh.f.Name,
Attributes: fh.f.attributes,
Chunks: fh.f.Chunks,
},
} }
glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.Chunks)) glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
for i, chunk := range fh.f.Chunks { for i, chunk := range fh.f.entry.Chunks {
glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
} }
if _, err := client.UpdateEntry(ctx, request); err != nil { if _, err := client.CreateEntry(ctx, request); err != nil {
return fmt.Errorf("update fh: %v", err) return fmt.Errorf("update fh: %v", err)
} }
return nil return nil
}) })
if err == nil {
fh.dirtyMetadata = false
}
return err
} }
func volumeId(fileId string) string { func volumeId(fileId string) string {