diff --git a/weed/storage/needle/compact_map.go b/weed/storage/needle/compact_map.go index 9852dca74..63781f6a2 100644 --- a/weed/storage/needle/compact_map.go +++ b/weed/storage/needle/compact_map.go @@ -117,14 +117,16 @@ func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset O x := cm.binarySearchCompactSection(key) if x < 0 { //println(x, "creating", len(cm.list), "section, starting", key) - cm.list = append(cm.list, NewCompactSection(key)) + cs := NewCompactSection(key) + cm.list = append(cm.list, cs) x = len(cm.list) - 1 //keep compact section sorted by start for x > 0 { - if cm.list[x-1].start > cm.list[x].start { - cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1] + if cm.list[x-1].start > key { + cm.list[x] = cm.list[x-1] x = x - 1 } else { + cm.list[x] = cs break } } diff --git a/weed/storage/needle/compact_map_perf_test.go b/weed/storage/needle/compact_map_perf_test.go index a66836ac8..5c73e01a9 100644 --- a/weed/storage/needle/compact_map_perf_test.go +++ b/weed/storage/needle/compact_map_perf_test.go @@ -1,15 +1,23 @@ package needle import ( - "log" - "os" "testing" + "os" + "log" - "github.com/chrislusf/seaweedfs/weed/glog" - . "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" + . "github.com/chrislusf/seaweedfs/weed/storage/types" ) +/* + +To see the memory usage: + +go test -run TestMemoryUsage -memprofile=mem.out +go tool pprof needle.test mem.out + + */ + func TestMemoryUsage(t *testing.T) { indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644) @@ -18,18 +26,16 @@ func TestMemoryUsage(t *testing.T) { } loadNewNeedleMap(indexFile) + indexFile.Close() + } func loadNewNeedleMap(file *os.File) { m := NewCompactMap() - bytes := make([]byte, 16*1024) + bytes := make([]byte, NeedleEntrySize*1024) count, e := file.Read(bytes) - if count > 0 { - fstat, _ := file.Stat() - glog.V(0).Infoln("Loading index file", fstat.Name(), "size", fstat.Size()) - } for count > 0 && e == nil { - for i := 0; i < count; i += 16 { + for i := 0; i < count; i += NeedleEntrySize { key := BytesToNeedleId(bytes[i : i+NeedleIdSize]) offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize]) size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize]) @@ -43,4 +49,38 @@ func loadNewNeedleMap(file *os.File) { count, e = file.Read(bytes) } + + m.report() + +} + +// report memory usage +func (cm *CompactMap) report() { + overFlowCount := 0; + overwrittenByOverflow := 0; + entryCount := 0 + compactSectionCount := 0 + compactSectionEntryCount := 0 + for _, cs := range cm.list { + compactSectionCount++ + cs.RLock() + for range cs.overflow { + overFlowCount++ + entryCount++ + } + for _, v := range cs.values { + compactSectionEntryCount++ + if _, found := cs.overflow[v.Key]; !found { + entryCount++ + } else { + overwrittenByOverflow++ + } + } + cs.RUnlock() + } + println("overFlowCount", overFlowCount) + println("overwrittenByOverflow", overwrittenByOverflow) + println("entryCount", entryCount) + println("compactSectionCount", compactSectionCount) + println("compactSectionEntryCount", compactSectionEntryCount) }