gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

refactoring

Browse source
This commit is contained in:
chrislu 2022-01-22 05:40:10 -08:00
parent d97bd54e63
commit 8aa6bf0bb9
2 changed files with 28 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
weed/filesys/page_writer/page_chunk.go
View file

@ -11,9 +11,9 @@ type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, cleanupF
type PageChunk interface { type PageChunk interface {
FreeResource() FreeResource()
WriteDataAt(src []byte, offset int64) (n int) WriteDataAt(src []byte, offset int64) (n int)
ReadDataAt(p []byte, off int64, logicChunkIndex LogicChunkIndex, chunkSize int64) (maxStop int64) ReadDataAt(p []byte, off int64) (maxStop int64)
IsComplete(chunkSize int64) bool IsComplete() bool
SaveContent(saveFn SaveToStorageFunc, logicChunkIndex LogicChunkIndex, chunkSize int64) SaveContent(saveFn SaveToStorageFunc)
} }
var ( var (
@ -23,6 +23,17 @@ var (
type MemChunk struct { type MemChunk struct {
buf []byte buf []byte
usage *ChunkWrittenIntervalList usage *ChunkWrittenIntervalList
chunkSize int64
logicChunkIndex LogicChunkIndex
}
func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk {
return &MemChunk{
logicChunkIndex: logicChunkIndex,
chunkSize: chunkSize,
buf: mem.Allocate(int(chunkSize)),
usage: newChunkWrittenIntervalList(),
}
} }
func (mc *MemChunk) FreeResource() { func (mc *MemChunk) FreeResource() {
@ -35,10 +46,10 @@ func (mc *MemChunk) WriteDataAt(src []byte, offset int64) (n int) {
return return
} }
func (mc *MemChunk) ReadDataAt(p []byte, off int64, logicChunkIndex LogicChunkIndex, chunkSize int64) (maxStop int64) { func (mc *MemChunk) ReadDataAt(p []byte, off int64) (maxStop int64) {
memChunkBaseOffset := int64(logicChunkIndex) * chunkSize memChunkBaseOffset := int64(mc.logicChunkIndex) * mc.chunkSize
for t := mc.usage.head.next; t != mc.usage.tail; t = t.next { for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
logicStart := max(off, int64(logicChunkIndex)*chunkSize+t.StartOffset) logicStart := max(off, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset)
logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset) logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
if logicStart < logicStop { if logicStart < logicStop {
copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset]) copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
@ -48,17 +59,17 @@ func (mc *MemChunk) ReadDataAt(p []byte, off int64, logicChunkIndex LogicChunkIn
return return
} }
func (mc *MemChunk) IsComplete(chunkSize int64) bool { func (mc *MemChunk) IsComplete() bool {
return mc.usage.IsComplete(chunkSize) return mc.usage.IsComplete(mc.chunkSize)
} }
func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc, logicChunkIndex LogicChunkIndex, chunkSize int64) { func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) {
if saveFn == nil { if saveFn == nil {
return return
} }
for t := mc.usage.head.next; t != mc.usage.tail; t = t.next { for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
reader := util.NewBytesReader(mc.buf[t.StartOffset:t.stopOffset]) reader := util.NewBytesReader(mc.buf[t.StartOffset:t.stopOffset])
saveFn(reader, int64(logicChunkIndex)*chunkSize+t.StartOffset, t.Size(), func() { saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), func() {
}) })
} }
} }

14
weed/filesys/page_writer/upload_pipeline.go
View file

@ -4,7 +4,6 @@ import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/mem"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
@ -62,10 +61,7 @@ func (cw *UploadPipeline) SaveDataAt(p []byte, off int64) (n int) {
memChunk, found := cw.writableChunks[logicChunkIndex] memChunk, found := cw.writableChunks[logicChunkIndex]
if !found { if !found {
memChunk = &MemChunk{ memChunk = NewMemChunk(logicChunkIndex, cw.ChunkSize)
buf: mem.Allocate(int(cw.ChunkSize)),
usage: newChunkWrittenIntervalList(),
}
cw.writableChunks[logicChunkIndex] = memChunk cw.writableChunks[logicChunkIndex] = memChunk
} }
n = memChunk.WriteDataAt(p, offsetRemainder) n = memChunk.WriteDataAt(p, offsetRemainder)
@ -85,7 +81,7 @@ func (cw *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
} }
cw.sealedChunksLock.Unlock() cw.sealedChunksLock.Unlock()
if found { if found {
maxStop = sealedChunk.chunk.ReadDataAt(p, off, logicChunkIndex, cw.ChunkSize) maxStop = sealedChunk.chunk.ReadDataAt(p, off)
glog.V(4).Infof("%s read sealed memchunk [%d,%d)", cw.filepath, off, maxStop) glog.V(4).Infof("%s read sealed memchunk [%d,%d)", cw.filepath, off, maxStop)
sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", cw.filepath, logicChunkIndex)) sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", cw.filepath, logicChunkIndex))
} }
@ -97,7 +93,7 @@ func (cw *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
if !found { if !found {
return return
} }
writableMaxStop := writableChunk.ReadDataAt(p, off, logicChunkIndex, cw.ChunkSize) writableMaxStop := writableChunk.ReadDataAt(p, off)
glog.V(4).Infof("%s read writable memchunk [%d,%d)", cw.filepath, off, writableMaxStop) glog.V(4).Infof("%s read writable memchunk [%d,%d)", cw.filepath, off, writableMaxStop)
maxStop = max(maxStop, writableMaxStop) maxStop = max(maxStop, writableMaxStop)
@ -174,7 +170,7 @@ func (cw *UploadPipeline) waitForCurrentWritersToComplete() {
} }
func (cw *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) { func (cw *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
if memChunk.IsComplete(cw.ChunkSize) { if memChunk.IsComplete() {
cw.moveToSealed(memChunk, logicChunkIndex) cw.moveToSealed(memChunk, logicChunkIndex)
} }
} }
@ -199,7 +195,7 @@ func (cw *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
cw.uploaders.Execute(func() { cw.uploaders.Execute(func() {
// first add to the file chunks // first add to the file chunks
sealedChunk.chunk.SaveContent(cw.saveToStorageFn, logicChunkIndex, cw.ChunkSize) sealedChunk.chunk.SaveContent(cw.saveToStorageFn)
// notify waiting process // notify waiting process
atomic.AddInt32(&cw.uploaderCount, -1) atomic.AddInt32(&cw.uploaderCount, -1)