gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

refactor

Browse source
This commit is contained in:
chrislu 2023-01-10 00:46:46 -08:00
parent 51d4a4b28d
commit 75bdd4a0d1
3 changed files with 68 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
weed/filer/filechunk_section.go
View file

@ -33,16 +33,7 @@ func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error {
if section.visibleIntervals != nil {
MergeIntoVisibles(section.visibleIntervals, start, stop, chunk)
garbageFileIds := FindGarbageChunks(section.visibleIntervals, start, stop)
for _, garbageFileId := range garbageFileIds {
length := len(section.chunks)
for i, t := range section.chunks {
if t.FileId == garbageFileId {
section.chunks[i] = section.chunks[length-1]
section.chunks = section.chunks[:length-1]
break
}
}
}
removeGarbageChunks(section, garbageFileIds)
}
if section.chunkViews != nil {
@ -52,6 +43,22 @@ func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error {
return nil
}
func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]struct{}) {
now := time.Now()
for i := 0; i < len(section.chunks); {
t := section.chunks[i]
length := len(section.chunks)
if _, found := garbageFileIds[t.FileId]; found {
if i < length-1 {
section.chunks[i] = section.chunks[length-1]
}
section.chunks = section.chunks[:length-1]
} else {
i++
}
}
}
func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64) {
if section.visibleIntervals == nil {
section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)

48
weed/filer/filechunk_section_test.go Normal file
View file

@ -0,0 +1,48 @@
package filer
import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"testing"
)
func Test_removeGarbageChunks(t *testing.T) {
section := NewFileChunkSection(0)
section.addChunk(&filer_pb.FileChunk{
FileId: "0",
Offset: 0,
Size: 1,
ModifiedTsNs: 0,
})
section.addChunk(&filer_pb.FileChunk{
FileId: "1",
Offset: 1,
Size: 1,
ModifiedTsNs: 1,
})
section.addChunk(&filer_pb.FileChunk{
FileId: "2",
Offset: 2,
Size: 1,
ModifiedTsNs: 2,
})
section.addChunk(&filer_pb.FileChunk{
FileId: "3",
Offset: 3,
Size: 1,
ModifiedTsNs: 3,
})
section.addChunk(&filer_pb.FileChunk{
FileId: "4",
Offset: 4,
Size: 1,
ModifiedTsNs: 4,
})
garbageFileIds := make(map[string]struct{})
garbageFileIds["0"] = struct{}{}
garbageFileIds["2"] = struct{}{}
garbageFileIds["4"] = struct{}{}
removeGarbageChunks(section, garbageFileIds)
if len(section.chunks) != 2 {
t.Errorf("remove chunk 2 failed")
}
}

5
weed/filer/filechunks.go
View file

@ -86,12 +86,13 @@ func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*f
return compacted, garbage
}
func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileId []string) {
func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileIds map[string]struct{}) {
garbageFileIds = make(map[string]struct{})
for x := visibles.Front(); x != nil; x = x.Next {
interval := x.Value
offset := interval.start - interval.offsetInChunk
if start <= offset && offset+int64(interval.chunkSize) <= stop {
garbageFileId = append(garbageFileId, interval.fileId)
garbageFileIds[interval.fileId] = struct{}{}
}
}
return