From 625f880b155eed2b1214a0490cf73c871a6ab18a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 26 May 2014 17:34:54 -0700 Subject: [PATCH] Adjust "weed server" parameters. --- go/storage/volume_vacuum.go | 9 ++++++++- go/weed/server.go | 5 +++-- go/weed/weed_server/master_server.go | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 3aeb11e64..9836a8dd9 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -4,6 +4,7 @@ import ( "code.google.com/p/weed-fs/go/glog" "fmt" "os" + "time" ) func (v *Volume) garbageLevel() float64 { @@ -11,16 +12,20 @@ func (v *Volume) garbageLevel() float64 { } func (v *Volume) Compact() error { + glog.V(3).Infof("Compacting ...") v.accessLock.Lock() defer v.accessLock.Unlock() + glog.V(3).Infof("Got Compaction lock...") filePath := v.FileName() glog.V(3).Infof("creating copies for volume %d ...", v.Id) return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx") } func (v *Volume) commitCompact() error { + glog.V(3).Infof("Committing vacuuming...") v.accessLock.Lock() defer v.accessLock.Unlock() + glog.V(3).Infof("Got Committing lock...") _ = v.dataFile.Close() var e error if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil { @@ -29,6 +34,8 @@ func (v *Volume) commitCompact() error { if e = os.Rename(v.FileName()+".cpx", v.FileName()+".idx"); e != nil { return e } + //glog.V(3).Infof("Pretending to be vacuuming...") + //time.Sleep(20 * time.Second) if e = v.load(true, false); e != nil { return e } @@ -57,7 +64,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err erro return err }, func(n *Needle, offset int64) error { nv, ok := v.nm.Get(n.Id) - glog.V(3).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv) + glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv) if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 { if _, err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil { return fmt.Errorf("cannot put needle: %s", err) diff --git a/go/weed/server.go b/go/weed/server.go index 80077a747..8b0372159 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -29,7 +29,7 @@ func init() { } var cmdServer = &Command{ - UsageLine: "server -port=8080 -dir=/tmp -max=5 -ip=server_name", + UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name", Short: "start a server, including volume server, and automatically elect a master server", Long: `start both a volume server to provide storage spaces and a master server to provide volume=>location mapping service and sequence number of file ids @@ -54,6 +54,7 @@ var ( serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name") serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") serverPeers = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list") + serverGarbageThreshold = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port") masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified") masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") @@ -174,7 +175,7 @@ func runServer(cmd *Command, args []string) bool { go func() { r := mux.NewRouter() ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder, - *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *garbageThreshold, serverWhiteList, + *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, serverWhiteList, ) glog.V(0).Infoln("Start Weed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort)) diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index 874688cbb..a89959fa4 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -5,7 +5,6 @@ import ( "code.google.com/p/weed-fs/go/sequence" "code.google.com/p/weed-fs/go/topology" "code.google.com/p/weed-fs/go/util" - "errors" "github.com/goraft/raft" "github.com/gorilla/mux" "net/http" @@ -110,7 +109,7 @@ func (ms *MasterServer) proxyToLeader(f func(w http.ResponseWriter, r *http.Requ proxy.ServeHTTP(w, r) } else { //drop it to the floor - writeJsonError(w, r, errors.New(ms.Topo.RaftServer.Name()+" does not know Leader yet:"+ms.Topo.RaftServer.Leader())) + //writeJsonError(w, r, errors.New(ms.Topo.RaftServer.Name()+" does not know Leader yet:"+ms.Topo.RaftServer.Leader())) } } }