From d61bea90385a73c9c500a9feb3dac267eb1c10d2 Mon Sep 17 00:00:00 2001 From: banjiaojuhao Date: Mon, 7 Mar 2022 15:14:24 +0800 Subject: [PATCH 1/2] [bugfix] filer: In file modification, old chunks will be mis-deleted when they are merged(Manifestized). --- weed/filer/filer_deletion.go | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/weed/filer/filer_deletion.go b/weed/filer/filer_deletion.go index e0191d7f1..48986a82a 100644 --- a/weed/filer/filer_deletion.go +++ b/weed/filer/filer_deletion.go @@ -129,6 +129,12 @@ func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) { } } +func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) { + for _, chunk := range chunks { + f.fileIdDeletionQueue.EnQueue(chunk.GetFileIdString()) + } +} + func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) { if oldEntry == nil { @@ -141,14 +147,36 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) { var toDelete []*filer_pb.FileChunk newChunkIds := make(map[string]bool) - for _, newChunk := range newEntry.Chunks { + newDChunks, newMChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), + newEntry.Chunks, 0, int64(newEntry.Size())) + if err != nil { + glog.Errorf("Failed to resolve new entry chunks when delete old entry chunks. new: %s, old: %s", + newEntry.Chunks, oldEntry.Chunks) + return + } + for _, newChunk := range newDChunks { + newChunkIds[newChunk.GetFileIdString()] = true + } + for _, newChunk := range newMChunks { newChunkIds[newChunk.GetFileIdString()] = true } - for _, oldChunk := range oldEntry.Chunks { + oldDChunks, oldMChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), + oldEntry.Chunks, 0, int64(oldEntry.Size())) + if err != nil { + glog.Errorf("Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s", + newEntry.Chunks, oldEntry.Chunks) + return + } + for _, oldChunk := range oldDChunks { if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found { toDelete = append(toDelete, oldChunk) } } - f.DeleteChunks(toDelete) + for _, oldChunk := range oldMChunks { + if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found { + toDelete = append(toDelete, oldChunk) + } + } + f.DeleteChunksNotRecursive(toDelete) } From f7f2a597dd8f56683f8acd3254a37fca5518eae5 Mon Sep 17 00:00:00 2001 From: banjiaojuhao Date: Tue, 8 Mar 2022 16:22:43 +0800 Subject: [PATCH 2/2] minor --- weed/filer/filechunk_manifest.go | 6 +++--- weed/filer/filer_deletion.go | 17 +++++++++-------- weed/shell/command_volume_fsck.go | 12 ++++++------ 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index 29860c27a..d036cfd24 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -68,12 +68,12 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun manifestChunks = append(manifestChunks, chunk) // recursive - dchunks, mchunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset) + dataChunks, manifestChunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset) if subErr != nil { return chunks, nil, subErr } - dataChunks = append(dataChunks, dchunks...) - manifestChunks = append(manifestChunks, mchunks...) + dataChunks = append(dataChunks, dataChunks...) + manifestChunks = append(manifestChunks, manifestChunks...) } return } diff --git a/weed/filer/filer_deletion.go b/weed/filer/filer_deletion.go index 48986a82a..e73f94151 100644 --- a/weed/filer/filer_deletion.go +++ b/weed/filer/filer_deletion.go @@ -1,6 +1,7 @@ package filer import ( + "math" "strings" "time" @@ -147,33 +148,33 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) { var toDelete []*filer_pb.FileChunk newChunkIds := make(map[string]bool) - newDChunks, newMChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), - newEntry.Chunks, 0, int64(newEntry.Size())) + newDataChunks, newManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), + newEntry.Chunks, 0, math.MaxInt64) if err != nil { glog.Errorf("Failed to resolve new entry chunks when delete old entry chunks. new: %s, old: %s", newEntry.Chunks, oldEntry.Chunks) return } - for _, newChunk := range newDChunks { + for _, newChunk := range newDataChunks { newChunkIds[newChunk.GetFileIdString()] = true } - for _, newChunk := range newMChunks { + for _, newChunk := range newManifestChunks { newChunkIds[newChunk.GetFileIdString()] = true } - oldDChunks, oldMChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), - oldEntry.Chunks, 0, int64(oldEntry.Size())) + oldDataChunks, oldManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(), + oldEntry.Chunks, 0, math.MaxInt64) if err != nil { glog.Errorf("Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s", newEntry.Chunks, oldEntry.Chunks) return } - for _, oldChunk := range oldDChunks { + for _, oldChunk := range oldDataChunks { if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found { toDelete = append(toDelete, oldChunk) } } - for _, oldChunk := range oldMChunks { + for _, oldChunk := range oldManifestChunks { if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found { toDelete = append(toDelete, oldChunk) } diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 28f2d6753..1b3d7bf0d 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -153,12 +153,12 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint if verbose && entry.Entry.IsDirectory { fmt.Fprintf(writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name)) } - dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) + dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) if resolveErr != nil { return nil } - dChunks = append(dChunks, mChunks...) - for _, chunk := range dChunks { + dataChunks = append(dataChunks, manifestChunks...) + for _, chunk := range dataChunks { outputChan <- &Item{ vid: chunk.Fid.VolumeId, fileKey: chunk.Fid.FileKey, @@ -332,15 +332,15 @@ func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInf fileKey uint64 } return doTraverseBfsAndSaving(c.env, nil, "/", false, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { - dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) + dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64) if resolveErr != nil { if verbose { fmt.Fprintf(writer, "resolving manifest chunks in %s: %v\n", util.NewFullPath(entry.Dir, entry.Entry.Name), resolveErr) } return nil } - dChunks = append(dChunks, mChunks...) - for _, chunk := range dChunks { + dataChunks = append(dataChunks, manifestChunks...) + for _, chunk := range dataChunks { outputChan <- &Item{ vid: chunk.Fid.VolumeId, fileKey: chunk.Fid.FileKey,