gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

merge chunks during upload (#4130)

Browse source 
* merge chunks during upload

* fix test
This commit is contained in:
Chris Lu 2023-01-12 23:07:36 -08:00 committed by GitHub
parent 2452f3b2f7
commit 1cd2e64aac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 31 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
weed/filer/filechunk_section_test.go
View file

@ -7,31 +7,31 @@ import (
func Test_removeGarbageChunks(t *testing.T) { func Test_removeGarbageChunks(t *testing.T) {
section := NewFileChunkSection(0) section := NewFileChunkSection(0)
section.addChunk(&filer_pb.FileChunk{ section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "0", FileId: "0",
Offset: 0, Offset: 0,
Size: 1, Size: 1,
ModifiedTsNs: 0, ModifiedTsNs: 0,
}) })
section.addChunk(&filer_pb.FileChunk{ section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "1", FileId: "1",
Offset: 1, Offset: 1,
Size: 1, Size: 1,
ModifiedTsNs: 1, ModifiedTsNs: 1,
}) })
section.addChunk(&filer_pb.FileChunk{ section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "2", FileId: "2",
Offset: 2, Offset: 2,
Size: 1, Size: 1,
ModifiedTsNs: 2, ModifiedTsNs: 2,
}) })
section.addChunk(&filer_pb.FileChunk{ section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "3", FileId: "3",
Offset: 3, Offset: 3,
Size: 1, Size: 1,
ModifiedTsNs: 3, ModifiedTsNs: 3,
}) })
section.addChunk(&filer_pb.FileChunk{ section.chunks = append(section.chunks, &filer_pb.FileChunk{
FileId: "4", FileId: "4",
Offset: 4, Offset: 4,
Size: 1, Size: 1,

14
weed/mount/page_writer/page_chunk_mem.go
View file

@ -105,4 +105,18 @@ func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) {
saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), t.TsNs, func() { saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), t.TsNs, func() {
}) })
} }
for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
startOffset := t.StartOffset
stopOffset := t.stopOffset
tsNs := t.TsNs
for t != mc.usage.tail && t.next.StartOffset == stopOffset {
stopOffset = t.next.stopOffset
t = t.next
tsNs = max(tsNs, t.TsNs)
}
reader := util.NewBytesReader(mc.buf[startOffset:stopOffset])
saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+startOffset, stopOffset-startOffset, tsNs, func() {
})
}
} }

15
weed/mount/page_writer/page_chunk_swapfile.go
View file

@ -176,11 +176,20 @@ func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
} }
// println(sc.logicChunkIndex, "|", "save") // println(sc.logicChunkIndex, "|", "save")
for t := sc.usage.head.next; t != sc.usage.tail; t = t.next { for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
data := mem.Allocate(int(t.Size())) startOffset := t.StartOffset
n, _ := sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize) stopOffset := t.stopOffset
tsNs := t.TsNs
for t != sc.usage.tail && t.next.StartOffset == stopOffset {
stopOffset = t.next.stopOffset
t = t.next
tsNs = max(tsNs, t.TsNs)
}
data := mem.Allocate(int(stopOffset - startOffset))
n, _ := sc.swapfile.file.ReadAt(data, startOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
if n > 0 { if n > 0 {
reader := util.NewBytesReader(data[:n]) reader := util.NewBytesReader(data[:n])
saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, int64(n), t.TsNs, func() { saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+startOffset, int64(n), tsNs, func() {
}) })
} }
mem.Free(data) mem.Free(data)