gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Reduce memory usage for "weed fix"

Browse source
This commit is contained in:
Chris Lu 2014-05-31 17:10:51 -07:00
parent 08cbb2c345
commit 4b7b439be9
4 changed files with 9 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
go/storage/volume.go
View file

@ -309,6 +309,7 @@ func (v *Volume) freeze() error {
func ScanVolumeFile(dirname string, collection string, id VolumeId,
visitSuperBlock func(SuperBlock) error,
readNeedleBody bool,
visitNeedle func(n *Needle, offset int64) error) (err error) {
var v *Volume
if v, err = loadVolumeWithoutIndex(dirname, collection, id); err != nil {
@ -327,9 +328,11 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
return
}
for n != nil {
if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil {
err = fmt.Errorf("cannot read needle body: %s", err)
return
if readNeedleBody {
if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil {
err = fmt.Errorf("cannot read needle body: %s", err)
return
}
}
if err = visitNeedle(n, offset); err != nil {
return

2
go/storage/volume_vacuum.go
View file

@ -62,7 +62,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err erro
err = ScanVolumeFile(v.dir, v.Collection, v.Id, func(superBlock SuperBlock) error {
_, err = dst.Write(superBlock.Bytes())
return err
}, func(n *Needle, offset int64) error {
}, true, func(n *Needle, offset int64) error {
nv, ok := v.nm.Get(n.Id)
glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {

2
go/weed/export.go
View file

@ -102,7 +102,7 @@ func runExport(cmd *Command, args []string) bool {
err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid, func(superBlock storage.SuperBlock) error {
version = superBlock.Version
return nil
}, func(n *storage.Needle, offset int64) error {
}, true, func(n *storage.Needle, offset int64) error {
nv, ok := nm.Get(n.Id)
glog.V(3).Infoln("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped(), "ok", ok, "nv", nv)
if ok && nv.Size > 0 {

2
go/weed/fix.go
View file

@ -49,7 +49,7 @@ func runFix(cmd *Command, args []string) bool {
vid := storage.VolumeId(*fixVolumeId)
err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, func(superBlock storage.SuperBlock) error {
return nil
}, func(n *storage.Needle, offset int64) error {
}, false, func(n *storage.Needle, offset int64) error {
debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())
if n.Size > 0 {
count, pe := nm.Put(n.Id, uint32(offset/storage.NeedlePaddingSize), n.Size)