From c4b92e17d033df1e348d3ab83c1ba5603e473a4f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 22:45:52 -0700 Subject: [PATCH] fix isPerfectAppend --- weed/filesys/dirty_page.go | 41 ++++++++++++++++++++++++++++---------- weed/filesys/filehandle.go | 14 ++++++------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 584db809d..04e05b440 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -29,21 +29,29 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da pages.Lock() defer pages.Unlock() - isPerfectAppend := len(pages.pages) == 0 + isPerfectOverwrite := false + isPerfectAppend := false if len(pages.pages) > 0 { lastPage := pages.pages[len(pages.pages)-1] if lastPage.Offset+int64(len(lastPage.Data)) == offset { // write continuous pages - glog.V(3).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) isPerfectAppend = true } + if pages.pages[0].Offset == offset && pages.totalSize() == int64(len(data)) { + glog.V(4).Infof("%s/%s overwrite [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + isPerfectOverwrite = true + } + } else { + glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + isPerfectAppend = true } isPerfectReplace := false for _, page := range pages.pages { if page.Offset == offset && len(page.Data) == len(data) { // perfect replace - glog.V(3).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + glog.V(4).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) page.Data = data isPerfectReplace = true } @@ -53,23 +61,34 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da return nil, nil } - if isPerfectAppend { - pages.pages = append(pages.pages, &DirtyPage{ - Offset: offset, - Data: data, - }) + if isPerfectAppend || isPerfectOverwrite { + if isPerfectAppend { + glog.V(4).Infof("%s/%s append2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + pages.pages = append(pages.pages, &DirtyPage{ + Offset: offset, + Data: data, + }) + } + + if isPerfectOverwrite { + glog.V(4).Infof("%s/%s overwrite2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + pages.pages = []*DirtyPage{&DirtyPage{ + Offset: offset, + Data: data, + }} + } if pages.f.wfs.chunkSizeLimit > 0 && pages.totalSize() >= pages.f.wfs.chunkSizeLimit { chunk, err = pages.saveToStorage(ctx) pages.pages = nil - glog.V(3).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) } return } chunk, err = pages.saveToStorage(ctx) - glog.V(3).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) pages.pages = []*DirtyPage{&DirtyPage{ Offset: offset, @@ -87,7 +106,7 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f if chunk, err = pages.saveToStorage(ctx); err == nil { pages.pages = nil if chunk != nil { - glog.V(3).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) } } return diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 52df2824c..9fa971236 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -39,7 +39,7 @@ var _ = fs.HandleReleaser(&FileHandle{}) func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - glog.V(3).Infof("%v/%v read fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(req.Size)) + glog.V(4).Infof("%v/%v read fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(req.Size)) if len(fh.f.Chunks) == 0 { glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name) @@ -73,7 +73,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus }) if err != nil { - glog.V(3).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err) + glog.V(4).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err) return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err) } @@ -84,7 +84,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus go func(chunkView *filer2.ChunkView) { defer wg.Done() - glog.V(3).Infof("read fh reading chunk: %+v", chunkView) + glog.V(4).Infof("read fh reading chunk: %+v", chunkView) locations := vid2Locations[volumeId(chunkView.FileId)] if locations == nil || len(locations.Locations) == 0 { @@ -109,7 +109,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus return } - glog.V(3).Infof("read fh read %d bytes: %+v", n, chunkView) + glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView) totalRead += n }(chunkView) @@ -126,7 +126,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f // write the request to volume servers - glog.V(3).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data))) + glog.V(4).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data))) chunk, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data) if err != nil { @@ -146,7 +146,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - glog.V(3).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name) + glog.V(4).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name) fh.f.isOpen = false @@ -158,7 +158,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { // fflush works at fh level // send the data to the OS - glog.V(3).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req) + glog.V(4).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req) chunk, err := fh.dirtyPages.FlushToStorage(ctx) if err != nil {