seaweedfs/go/topology/topology_compact.go

153 lines
4.4 KiB
Go
Raw Normal View History

2012-11-07 09:51:43 +00:00
package topology
import (
"code.google.com/p/weed-fs/go/glog"
2013-02-27 06:54:22 +00:00
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/util"
2012-11-07 09:51:43 +00:00
"encoding/json"
"errors"
"net/url"
"time"
)
func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool {
2012-11-24 01:03:27 +00:00
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url string, vid storage.VolumeId) {
//glog.V(0).Infoln(index, "Check vacuuming", vid, "on", dn.Url())
if e, ret := vacuumVolume_Check(url, vid, garbageThreshold); e != nil {
//glog.V(0).Infoln(index, "Error when checking vacuuming", vid, "on", url, e)
2012-11-24 01:03:27 +00:00
ch <- false
} else {
//glog.V(0).Infoln(index, "Checked vacuuming", vid, "on", url, "needVacuum", ret)
2012-11-24 01:03:27 +00:00
ch <- ret
}
}(index, dn.Url(), vid)
}
isCheckSuccess := true
for _ = range locationlist.list {
select {
case canVacuum := <-ch:
isCheckSuccess = isCheckSuccess && canVacuum
case <-time.After(30 * time.Minute):
isCheckSuccess = false
break
}
}
return isCheckSuccess
}
func batchVacuumVolumeCompact(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
vl.removeFromWritable(vid)
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url string, vid storage.VolumeId) {
glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
2012-11-24 01:03:27 +00:00
if e := vacuumVolume_Compact(url, vid); e != nil {
glog.V(0).Infoln(index, "Error when vacuuming", vid, "on", url, e)
2012-11-24 01:03:27 +00:00
ch <- false
} else {
glog.V(0).Infoln(index, "Complete vacuuming", vid, "on", url)
2012-11-24 01:03:27 +00:00
ch <- true
}
}(index, dn.Url(), vid)
}
isVacuumSuccess := true
for _ = range locationlist.list {
select {
case _ = <-ch:
case <-time.After(30 * time.Minute):
isVacuumSuccess = false
break
}
}
return isVacuumSuccess
}
func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
isCommitSuccess := true
for _, dn := range locationlist.list {
glog.V(0).Infoln("Start Commiting vacuum", vid, "on", dn.Url())
2012-11-24 01:03:27 +00:00
if e := vacuumVolume_Commit(dn.Url(), vid); e != nil {
glog.V(0).Infoln("Error when committing vacuum", vid, "on", dn.Url(), e)
2012-11-24 01:03:27 +00:00
isCommitSuccess = false
} else {
glog.V(0).Infoln("Complete Commiting vacuum", vid, "on", dn.Url())
2012-11-24 01:03:27 +00:00
}
if isCommitSuccess {
vl.SetVolumeAvailable(dn, vid)
}
2012-11-24 01:03:27 +00:00
}
return isCommitSuccess
}
func (t *Topology) Vacuum(garbageThreshold string) int {
2013-11-12 10:21:22 +00:00
for _, c := range t.collectionMap {
for _, vl := range c.replicaType2VolumeLayout {
if vl != nil {
for vid, locationlist := range vl.vid2location {
if batchVacuumVolumeCheck(vl, vid, locationlist, garbageThreshold) {
if batchVacuumVolumeCompact(vl, vid, locationlist) {
batchVacuumVolumeCommit(vl, vid, locationlist)
}
2012-11-07 09:51:43 +00:00
}
}
}
}
}
return 0
}
type VacuumVolumeResult struct {
2012-11-24 01:03:27 +00:00
Result bool
Error string
2012-11-07 09:51:43 +00:00
}
func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThreshold string) (error, bool) {
2012-11-24 01:03:27 +00:00
values := make(url.Values)
values.Add("volume", vid.String())
2013-01-17 08:56:56 +00:00
values.Add("garbageThreshold", garbageThreshold)
2012-11-24 01:03:27 +00:00
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_check", values)
if err != nil {
glog.V(0).Infoln("parameters:", values)
2012-11-24 01:03:27 +00:00
return err, false
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err, false
}
if ret.Error != "" {
return errors.New(ret.Error), false
}
return nil, ret.Result
}
func vacuumVolume_Compact(urlLocation string, vid storage.VolumeId) error {
2012-11-07 09:51:43 +00:00
values := make(url.Values)
values.Add("volume", vid.String())
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_compact", values)
if err != nil {
2012-11-24 01:03:27 +00:00
return err
2012-11-07 09:51:43 +00:00
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
2012-11-24 01:03:27 +00:00
return err
2012-11-07 09:51:43 +00:00
}
if ret.Error != "" {
2012-11-24 01:03:27 +00:00
return errors.New(ret.Error)
2012-11-07 09:51:43 +00:00
}
2012-11-24 01:03:27 +00:00
return nil
2012-11-07 09:51:43 +00:00
}
func vacuumVolume_Commit(urlLocation string, vid storage.VolumeId) error {
values := make(url.Values)
values.Add("volume", vid.String())
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_commit", values)
if err != nil {
return err
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err
}
if ret.Error != "" {
return errors.New(ret.Error)
}
return nil
}