From 6ccd7f0a4d8ff5167054147cc66774fec84d80b6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 18 Aug 2020 18:01:37 -0700 Subject: [PATCH] refactoring --- weed/storage/needle_map/compact_map.go | 2 +- weed/storage/needle_map/compact_map_test.go | 4 ++-- weed/storage/needle_map_leveldb.go | 2 +- weed/storage/needle_map_memory.go | 4 ++-- weed/storage/needle_map_metric.go | 6 +++--- weed/storage/types/needle_types.go | 3 +++ weed/storage/volume_backup.go | 2 +- weed/storage/volume_read_write.go | 6 +++--- weed/storage/volume_vacuum.go | 4 ++-- 9 files changed, 18 insertions(+), 15 deletions(-) diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go index b8d242e2d..c1fb00268 100644 --- a/weed/storage/needle_map/compact_map.go +++ b/weed/storage/needle_map/compact_map.go @@ -130,7 +130,7 @@ func (cs *CompactSection) Delete(key NeedleId) Size { cs.Lock() ret := Size(0) if i := cs.binarySearchValues(skey); i >= 0 { - if cs.values[i].Size > 0 && cs.values[i].Size != TombstoneFileSize { + if cs.values[i].Size > 0 && cs.values[i].Size.IsValid() { ret = cs.values[i].Size cs.values[i].Size = TombstoneFileSize } diff --git a/weed/storage/needle_map/compact_map_test.go b/weed/storage/needle_map/compact_map_test.go index 33ed4f1ce..c6bfb97b4 100644 --- a/weed/storage/needle_map/compact_map_test.go +++ b/weed/storage/needle_map/compact_map_test.go @@ -76,7 +76,7 @@ func TestCompactMap(t *testing.T) { t.Fatal("key", i, "size", v.Size) } } else if i%37 == 0 { - if ok && v.Size != TombstoneFileSize { + if ok && v.Size.IsValid() { t.Fatal("key", i, "should have been deleted needle value", v) } } else if i%2 == 0 { @@ -89,7 +89,7 @@ func TestCompactMap(t *testing.T) { for i := uint32(10 * batch); i < 100*batch; i++ { v, ok := m.Get(NeedleId(i)) if i%37 == 0 { - if ok && v.Size != TombstoneFileSize { + if ok && v.Size.IsValid() { t.Fatal("key", i, "should have been deleted needle value", v) } } else if i%2 == 0 { diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go index b4b04c07f..4a17e6d0e 100644 --- a/weed/storage/needle_map_leveldb.go +++ b/weed/storage/needle_map_leveldb.go @@ -74,7 +74,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error { } defer db.Close() return idx.WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size Size) error { - if !offset.IsZero() && size != TombstoneFileSize { + if !offset.IsZero() && size.IsValid() { levelDbWrite(db, key, offset, size) } else { levelDbDelete(db, key) diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go index 8e7e51973..d0891dc98 100644 --- a/weed/storage/needle_map_memory.go +++ b/weed/storage/needle_map_memory.go @@ -30,11 +30,11 @@ func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) { func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) { e := idx.WalkIndexFile(file, func(key NeedleId, offset Offset, size Size) error { nm.MaybeSetMaxFileKey(key) - if !offset.IsZero() && size != TombstoneFileSize { + if !offset.IsZero() && size.IsValid() { nm.FileCounter++ nm.FileByteCounter = nm.FileByteCounter + uint64(size) oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size) - if !oldOffset.IsZero() && oldSize != TombstoneFileSize { + if !oldOffset.IsZero() && oldSize.IsValid() { nm.DeletionCounter++ nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) } diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.go index 85addc27d..3618dada9 100644 --- a/weed/storage/needle_map_metric.go +++ b/weed/storage/needle_map_metric.go @@ -31,7 +31,7 @@ func (mm *mapMetric) logPut(key NeedleId, oldSize Size, newSize Size) { } mm.MaybeSetMaxFileKey(key) mm.LogFileCounter(newSize) - if oldSize > 0 && oldSize != TombstoneFileSize { + if oldSize > 0 && oldSize.IsValid() { mm.LogDeletionCounter(oldSize) } } @@ -101,7 +101,7 @@ func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) { mm.MaybeSetMaxFileKey(key) NeedleIdToBytes(buf, key) - if size != TombstoneFileSize { + if size.IsValid() { mm.FileByteCounter += uint64(size) } @@ -111,7 +111,7 @@ func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) { } else { // deleted file mm.DeletionCounter++ - if size != TombstoneFileSize { + if size.IsValid() { // previously already deleted file mm.DeletionByteCounter += uint64(size) } diff --git a/weed/storage/types/needle_types.go b/weed/storage/types/needle_types.go index 138643f7f..0e9115c0d 100644 --- a/weed/storage/types/needle_types.go +++ b/weed/storage/types/needle_types.go @@ -18,6 +18,9 @@ type Size uint32 func (s Size) IsDeleted() bool { return s == TombstoneFileSize } +func (s Size) IsValid() bool { + return s != TombstoneFileSize +} type OffsetLower struct { b3 byte diff --git a/weed/storage/volume_backup.go b/weed/storage/volume_backup.go index f7075fe2b..595bd8a35 100644 --- a/weed/storage/volume_backup.go +++ b/weed/storage/volume_backup.go @@ -253,7 +253,7 @@ func (scanner *VolumeFileScanner4GenIdx) ReadNeedleBody() bool { } func (scanner *VolumeFileScanner4GenIdx) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error { - if n.Size > 0 && n.Size != TombstoneFileSize { + if n.Size > 0 && n.Size.IsValid() { return scanner.v.nm.Put(n.Id, ToOffset(offset), n.Size) } return scanner.v.nm.Delete(n.Id, ToOffset(offset)) diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go index 31be0640a..b2487dde0 100644 --- a/weed/storage/volume_read_write.go +++ b/weed/storage/volume_read_write.go @@ -25,7 +25,7 @@ func (v *Volume) isFileUnchanged(n *needle.Needle) bool { } nv, ok := v.nm.Get(n.Id) - if ok && !nv.Offset.IsZero() && nv.Size != TombstoneFileSize { + if ok && !nv.Offset.IsZero() && nv.Size.IsValid() { oldNeedle := new(needle.Needle) err := oldNeedle.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset(), nv.Size, v.Version()) if err != nil { @@ -196,7 +196,7 @@ func (v *Volume) syncDelete(n *needle.Needle) (Size, error) { nv, ok := v.nm.Get(n.Id) //fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size) - if ok && nv.Size != TombstoneFileSize { + if ok && nv.Size.IsValid() { size := nv.Size n.Data = nil n.AppendAtNs = uint64(time.Now().UnixNano()) @@ -234,7 +234,7 @@ func (v *Volume) doDeleteRequest(n *needle.Needle) (Size, error) { glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String()) nv, ok := v.nm.Get(n.Id) //fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size) - if ok && nv.Size != TombstoneFileSize { + if ok && nv.Size.IsValid() { size := nv.Size n.Data = nil n.AppendAtNs = uint64(time.Now().UnixNano()) diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index 9d366a27d..a3e5800df 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -274,7 +274,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI } //updated needle - if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size != TombstoneFileSize { + if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size.IsValid() { //even the needle cache in memory is hit, the need_bytes is correct glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size) var needleBytes []byte @@ -335,7 +335,7 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset in } nv, ok := scanner.v.nm.Get(n.Id) glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv) - if ok && nv.Offset.ToAcutalOffset() == offset && nv.Size > 0 && nv.Size != TombstoneFileSize { + if ok && nv.Offset.ToAcutalOffset() == offset && nv.Size > 0 && nv.Size.IsValid() { if err := scanner.nm.Set(n.Id, ToOffset(scanner.newOffset), n.Size); err != nil { return fmt.Errorf("cannot put needle: %s", err) }