seaweedfs/weed/topology/topology_vacuum.go

165 lines
4.8 KiB
Go
Raw Normal View History

2012-11-07 09:51:43 +00:00
package topology
import (
"encoding/json"
"errors"
"net/url"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
2012-11-07 09:51:43 +00:00
)
func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool {
2012-11-24 01:03:27 +00:00
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url string, vid storage.VolumeId) {
//glog.V(0).Infoln(index, "Check vacuuming", vid, "on", dn.Url())
if e, ret := vacuumVolume_Check(url, vid, garbageThreshold); e != nil {
//glog.V(0).Infoln(index, "Error when checking vacuuming", vid, "on", url, e)
2012-11-24 01:03:27 +00:00
ch <- false
} else {
//glog.V(0).Infoln(index, "Checked vacuuming", vid, "on", url, "needVacuum", ret)
2012-11-24 01:03:27 +00:00
ch <- ret
}
}(index, dn.Url(), vid)
2012-11-24 01:03:27 +00:00
}
isCheckSuccess := true
for _ = range locationlist.list {
select {
case canVacuum := <-ch:
isCheckSuccess = isCheckSuccess && canVacuum
case <-time.After(30 * time.Minute):
isCheckSuccess = false
break
}
}
return isCheckSuccess
}
func batchVacuumVolumeCompact(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
vl.removeFromWritable(vid)
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url string, vid storage.VolumeId) {
glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
2012-11-24 01:03:27 +00:00
if e := vacuumVolume_Compact(url, vid); e != nil {
glog.V(0).Infoln(index, "Error when vacuuming", vid, "on", url, e)
2012-11-24 01:03:27 +00:00
ch <- false
} else {
glog.V(0).Infoln(index, "Complete vacuuming", vid, "on", url)
2012-11-24 01:03:27 +00:00
ch <- true
}
}(index, dn.Url(), vid)
2012-11-24 01:03:27 +00:00
}
isVacuumSuccess := true
for _ = range locationlist.list {
select {
case _ = <-ch:
case <-time.After(30 * time.Minute):
isVacuumSuccess = false
break
}
}
return isVacuumSuccess
}
func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
isCommitSuccess := true
for _, dn := range locationlist.list {
glog.V(0).Infoln("Start Commiting vacuum", vid, "on", dn.Url())
if e := vacuumVolume_Commit(dn.Url(), vid); e != nil {
glog.V(0).Infoln("Error when committing vacuum", vid, "on", dn.Url(), e)
2012-11-24 01:03:27 +00:00
isCommitSuccess = false
} else {
glog.V(0).Infoln("Complete Commiting vacuum", vid, "on", dn.Url())
2012-11-24 01:03:27 +00:00
}
if isCommitSuccess {
vl.SetVolumeAvailable(dn, vid)
}
2012-11-24 01:03:27 +00:00
}
return isCommitSuccess
}
func (t *Topology) Vacuum(garbageThreshold string) int {
2016-06-26 19:49:08 +00:00
glog.V(0).Infof("Start vacuum on demand with threshold:%s", garbageThreshold)
2016-05-30 19:30:26 +00:00
for _, col := range t.collectionMap.Items() {
c := col.(*Collection)
2016-05-30 19:30:26 +00:00
for _, vl := range c.storageType2VolumeLayout.Items() {
2013-11-12 10:21:22 +00:00
if vl != nil {
volumeLayout := vl.(*VolumeLayout)
writableSet := make(map[storage.VolumeId]bool)
for _, id := range volumeLayout.writables {
writableSet[id] = true
}
for vid, locationlist := range volumeLayout.vid2location {
if _, isWritable := writableSet[vid]; !isWritable {
continue
}
2016-06-26 19:49:08 +00:00
glog.V(0).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
if batchVacuumVolumeCheck(volumeLayout, vid, locationlist, garbageThreshold) {
if batchVacuumVolumeCompact(volumeLayout, vid, locationlist) {
batchVacuumVolumeCommit(volumeLayout, vid, locationlist)
2013-11-12 10:21:22 +00:00
}
2012-11-07 09:51:43 +00:00
}
}
}
}
}
return 0
}
type VacuumVolumeResult struct {
2012-11-24 01:03:27 +00:00
Result bool
Error string
2012-11-07 09:51:43 +00:00
}
func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThreshold string) (error, bool) {
2012-11-24 01:03:27 +00:00
values := make(url.Values)
values.Add("volume", vid.String())
2013-01-17 08:56:56 +00:00
values.Add("garbageThreshold", garbageThreshold)
2015-05-17 20:19:39 +00:00
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum/check", values)
2012-11-24 01:03:27 +00:00
if err != nil {
glog.V(0).Infoln("parameters:", values)
2012-11-24 01:03:27 +00:00
return err, false
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err, false
}
if ret.Error != "" {
return errors.New(ret.Error), false
}
return nil, ret.Result
}
func vacuumVolume_Compact(urlLocation string, vid storage.VolumeId) error {
2012-11-07 09:51:43 +00:00
values := make(url.Values)
values.Add("volume", vid.String())
2015-05-17 20:19:39 +00:00
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum/compact", values)
2012-11-07 09:51:43 +00:00
if err != nil {
2012-11-24 01:03:27 +00:00
return err
2012-11-07 09:51:43 +00:00
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
2012-11-24 01:03:27 +00:00
return err
2012-11-07 09:51:43 +00:00
}
if ret.Error != "" {
2012-11-24 01:03:27 +00:00
return errors.New(ret.Error)
2012-11-07 09:51:43 +00:00
}
2012-11-24 01:03:27 +00:00
return nil
2012-11-07 09:51:43 +00:00
}
func vacuumVolume_Commit(urlLocation string, vid storage.VolumeId) error {
values := make(url.Values)
values.Add("volume", vid.String())
2015-05-17 20:19:39 +00:00
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum/commit", values)
2012-11-07 09:51:43 +00:00
if err != nil {
return err
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err
}
if ret.Error != "" {
return errors.New(ret.Error)
}
return nil
}