seaweedfs/weed/topology/topology_vacuum.go

239 lines
7.9 KiB
Go
Raw Normal View History

2012-11-07 09:51:43 +00:00
package topology
import (
2018-10-15 06:12:43 +00:00
"context"
"github.com/seaweedfs/seaweedfs/weed/pb"
"io"
"sync/atomic"
2012-11-07 09:51:43 +00:00
"time"
2019-04-19 04:43:36 +00:00
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
2012-11-07 09:51:43 +00:00
)
func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid needle.VolumeId,
locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
ch := make(chan int, locationlist.Length())
errCount := int32(0)
2012-11-24 01:03:27 +00:00
for index, dn := range locationlist.list {
go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
err := operation.WithVolumeServerClient(false, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
VolumeId: uint32(vid),
2018-10-15 06:12:43 +00:00
})
if err != nil {
atomic.AddInt32(&errCount, 1)
ch <- -1
2018-10-15 06:12:43 +00:00
return err
}
if resp.GarbageRatio >= garbageThreshold {
ch <- index
} else {
ch <- -1
}
2018-10-15 06:12:43 +00:00
return nil
})
if err != nil {
glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
2012-11-24 01:03:27 +00:00
}
}(index, dn.ServerAddress(), vid)
2012-11-24 01:03:27 +00:00
}
vacuumLocationList := NewVolumeLocationList()
2020-10-19 21:24:57 +00:00
waitTimeout := time.NewTimer(time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
2020-10-19 21:24:57 +00:00
defer waitTimeout.Stop()
for range locationlist.list {
2012-11-24 01:03:27 +00:00
select {
case index := <-ch:
if index != -1 {
vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
}
2020-10-19 21:24:57 +00:00
case <-waitTimeout.C:
return vacuumLocationList, false
2012-11-24 01:03:27 +00:00
}
}
return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
2012-11-24 01:03:27 +00:00
}
func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
locationlist *VolumeLocationList, preallocate int64) bool {
2019-07-21 20:49:09 +00:00
vl.accessLock.Lock()
2012-11-24 01:03:27 +00:00
vl.removeFromWritable(vid)
2019-07-21 20:49:09 +00:00
vl.accessLock.Unlock()
2012-11-24 01:03:27 +00:00
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
err := operation.WithVolumeServerClient(true, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
stream, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
VolumeId: uint32(vid),
Preallocate: preallocate,
2018-10-15 06:12:43 +00:00
})
if err != nil {
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr != nil {
if recvErr == io.EOF {
break
} else {
return recvErr
}
}
2022-08-01 16:32:21 +00:00
glog.V(0).Infof("%d vacuum %d on %s processed %d bytes, loadAvg %.02f%%",
index, vid, url, resp.ProcessedBytes, resp.LoadAvg_1M*100)
}
return nil
2018-10-15 06:12:43 +00:00
})
if err != nil {
glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
2012-11-24 01:03:27 +00:00
ch <- false
} else {
2018-10-15 06:12:43 +00:00
glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
2012-11-24 01:03:27 +00:00
ch <- true
}
}(index, dn.ServerAddress(), vid)
2012-11-24 01:03:27 +00:00
}
isVacuumSuccess := true
2020-10-19 21:24:57 +00:00
waitTimeout := time.NewTimer(3 * time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
2020-10-19 21:24:57 +00:00
defer waitTimeout.Stop()
for range locationlist.list {
2012-11-24 01:03:27 +00:00
select {
2017-08-30 06:11:08 +00:00
case canCommit := <-ch:
isVacuumSuccess = isVacuumSuccess && canCommit
2020-10-19 21:24:57 +00:00
case <-waitTimeout.C:
return false
2012-11-24 01:03:27 +00:00
}
}
return isVacuumSuccess
}
func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
2012-11-24 01:03:27 +00:00
isCommitSuccess := true
isReadOnly := false
2012-11-24 01:03:27 +00:00
for _, dn := range locationlist.list {
2019-02-06 13:59:15 +00:00
glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
resp, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
VolumeId: uint32(vid),
2018-10-15 06:12:43 +00:00
})
2020-12-02 08:09:19 +00:00
if resp != nil && resp.IsReadOnly {
isReadOnly = true
}
2018-10-15 06:12:43 +00:00
return err
})
if err != nil {
glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
2012-11-24 01:03:27 +00:00
isCommitSuccess = false
} else {
2019-02-06 13:59:15 +00:00
glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
2012-11-24 01:03:27 +00:00
}
}
if isCommitSuccess {
for _, dn := range locationlist.list {
vl.SetVolumeAvailable(dn, vid, isReadOnly)
}
2012-11-24 01:03:27 +00:00
}
return isCommitSuccess
}
func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
for _, dn := range locationlist.list {
glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
VolumeId: uint32(vid),
2018-10-15 06:12:43 +00:00
})
return err
})
if err != nil {
glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
} else {
2018-10-15 06:12:43 +00:00
glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
}
}
}
func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, volumeId uint32, collection string, preallocate int64) {
// if there is vacuum going on, return immediately
swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
if !swapped {
2020-11-29 07:18:02 +00:00
return
}
defer atomic.StoreInt64(&t.vacuumLockCounter, 0)
// now only one vacuum process going on
glog.V(1).Infof("Start vacuum on demand with threshold: %f collection: %s volumeId: %d",
garbageThreshold, collection, volumeId)
2016-05-30 19:30:26 +00:00
for _, col := range t.collectionMap.Items() {
c := col.(*Collection)
if collection != "" && collection != c.Name {
continue
}
2016-05-30 19:30:26 +00:00
for _, vl := range c.storageType2VolumeLayout.Items() {
2013-11-12 10:21:22 +00:00
if vl != nil {
volumeLayout := vl.(*VolumeLayout)
2022-07-08 17:29:24 +00:00
if volumeId > 0 {
vid := needle.VolumeId(volumeId)
volumeLayout.accessLock.RLock()
locationList, ok := volumeLayout.vid2location[vid]
volumeLayout.accessLock.RUnlock()
if ok {
t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate)
2022-07-08 17:29:24 +00:00
}
} else {
t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
2022-04-18 14:36:14 +00:00
}
2018-10-19 03:34:43 +00:00
}
}
}
}
2022-04-18 14:36:14 +00:00
func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {
2018-10-19 03:34:43 +00:00
volumeLayout.accessLock.RLock()
2019-04-19 04:43:36 +00:00
tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
2019-01-17 01:17:19 +00:00
for vid, locationList := range volumeLayout.vid2location {
tmpMap[vid] = locationList.Copy()
2018-10-19 03:34:43 +00:00
}
volumeLayout.accessLock.RUnlock()
2019-01-17 01:17:19 +00:00
for vid, locationList := range tmpMap {
t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate)
}
}
func (t *Topology) vacuumOneVolumeId(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, locationList *VolumeLocationList, vid needle.VolumeId, preallocate int64) {
volumeLayout.accessLock.RLock()
isReadOnly := volumeLayout.readonlyVolumes.IsTrue(vid)
isEnoughCopies := volumeLayout.enoughCopies(vid)
volumeLayout.accessLock.RUnlock()
2018-10-19 03:34:43 +00:00
if isReadOnly || !isEnoughCopies {
return
}
2018-10-19 03:34:43 +00:00
glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(
grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
if t.batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
t.batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
} else {
t.batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
2012-11-07 09:51:43 +00:00
}
}
}