mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
volume: fix memory leak during compaction
fix https://github.com/chrislusf/seaweedfs/issues/1222
This commit is contained in:
parent
1f8fc4b5ed
commit
0871d2cff0
|
@ -195,6 +195,8 @@ func runExport(cmd *Command, args []string) bool {
|
|||
vid := needle.VolumeId(*export.volumeId)
|
||||
|
||||
needleMap := needle_map.NewMemDb()
|
||||
defer needleMap.Close()
|
||||
|
||||
if err := needleMap.LoadFromIdx(path.Join(*export.dir, fileName+".idx")); err != nil {
|
||||
glog.Fatalf("cannot load needle map from %s.idx: %s", fileName, err)
|
||||
}
|
||||
|
|
|
@ -70,6 +70,7 @@ func runFix(cmd *Command, args []string) bool {
|
|||
indexFileName := path.Join(*fixVolumePath, baseFileName+".idx")
|
||||
|
||||
nm := needle_map.NewMemDb()
|
||||
defer nm.Close()
|
||||
|
||||
vid := needle.VolumeId(*fixVolumeId)
|
||||
scanner := &VolumeFileScanner4Fix{
|
||||
|
|
|
@ -61,7 +61,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque
|
|||
return
|
||||
}
|
||||
}
|
||||
glog.Infoln("garbageThreshold =", gcThreshold)
|
||||
// glog.Infoln("garbageThreshold =", gcThreshold)
|
||||
ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize)
|
||||
ms.dirStatusHandler(w, r)
|
||||
}
|
||||
|
|
|
@ -27,6 +27,9 @@ const (
|
|||
func WriteSortedFileFromIdx(baseFileName string, ext string) (e error) {
|
||||
|
||||
nm, err := readNeedleMap(baseFileName)
|
||||
if nm != nil {
|
||||
defer nm.Close()
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("readNeedleMap: %v", err)
|
||||
}
|
||||
|
@ -196,7 +199,7 @@ func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSi
|
|||
}
|
||||
|
||||
buffers := make([][]byte, TotalShardsCount)
|
||||
for i, _ := range buffers {
|
||||
for i := range buffers {
|
||||
buffers[i] = make([]byte, bufferSize)
|
||||
}
|
||||
|
||||
|
@ -233,7 +236,7 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o
|
|||
}
|
||||
|
||||
buffers := make([][]byte, TotalShardsCount)
|
||||
for i, _ := range buffers {
|
||||
for i := range buffers {
|
||||
if shardHasData[i] {
|
||||
buffers[i] = make([]byte, ErasureCodingSmallBlockSize)
|
||||
}
|
||||
|
|
|
@ -7,9 +7,10 @@ import (
|
|||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/reedsolomon"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -42,6 +43,7 @@ func TestEncodingDecoding(t *testing.T) {
|
|||
|
||||
func validateFiles(baseFileName string) error {
|
||||
nm, err := readNeedleMap(baseFileName)
|
||||
defer nm.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("readNeedleMap: %v", err)
|
||||
}
|
||||
|
|
|
@ -113,3 +113,7 @@ func (cm *MemDb) LoadFromIdx(idxName string) (ret error) {
|
|||
})
|
||||
|
||||
}
|
||||
|
||||
func (cm *MemDb) Close() {
|
||||
cm.db.Close()
|
||||
}
|
||||
|
|
23
weed/storage/needle_map/memdb_test.go
Normal file
23
weed/storage/needle_map/memdb_test.go
Normal file
|
@ -0,0 +1,23 @@
|
|||
package needle_map
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func BenchmarkMemDb(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
nm := NewMemDb()
|
||||
|
||||
nid := types.NeedleId(345)
|
||||
offset := types.Offset{
|
||||
OffsetHigher: types.OffsetHigher{},
|
||||
OffsetLower: types.OffsetLower{},
|
||||
}
|
||||
nm.Set(nid, offset, 324)
|
||||
nm.Close()
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in a new issue