seaweedfs/weed/shell/command_volume_balance.go

250 lines
7.3 KiB
Go
Raw Permalink Normal View History

2019-05-06 04:17:23 +00:00
package shell
import (
"context"
"flag"
"fmt"
"io"
"os"
"sort"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
func init() {
Commands = append(Commands, &commandVolumeBalance{})
2019-05-06 04:17:23 +00:00
}
type commandVolumeBalance struct {
}
func (c *commandVolumeBalance) Name() string {
return "volume.balance"
}
func (c *commandVolumeBalance) Help() string {
return `balance all volumes among volume servers
2019-11-25 06:17:43 +00:00
volume.balance [-collection ALL|EACH_COLLECTION|<collection_name>] [-force] [-dataCenter=<data_center_name>]
2019-05-06 04:17:23 +00:00
Algorithm:
2019-05-07 21:02:01 +00:00
2019-05-06 04:17:23 +00:00
For each type of volume server (different max volume count limit){
for each collection {
balanceWritableVolumes()
balanceReadOnlyVolumes()
}
2019-05-06 04:17:23 +00:00
}
func balanceWritableVolumes(){
idealWritableVolumes = totalWritableVolumes / numVolumeServers
2019-06-11 04:32:56 +00:00
for hasMovedOneVolume {
sort all volume servers ordered by the number of local writable volumes
2019-05-06 04:17:23 +00:00
pick the volume server A with the lowest number of writable volumes x
pick the volume server B with the highest number of writable volumes y
if y > idealWritableVolumes and x +1 <= idealWritableVolumes {
2019-05-06 04:17:23 +00:00
if B has a writable volume id v that A does not have {
move writable volume v from A to B
}
}
}
}
func balanceReadOnlyVolumes(){
//similar to balanceWritableVolumes
}
`
}
func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
2019-05-06 04:17:23 +00:00
balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
2019-06-03 09:26:31 +00:00
collection := balanceCommand.String("collection", "EACH_COLLECTION", "collection name, or use \"ALL_COLLECTIONS\" across collections, \"EACH_COLLECTION\" for each collection")
dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter")
2019-06-03 09:26:31 +00:00
applyBalancing := balanceCommand.Bool("force", false, "apply the balancing plan.")
2019-05-06 04:17:23 +00:00
if err = balanceCommand.Parse(args); err != nil {
return nil
}
var resp *master_pb.VolumeListResponse
ctx := context.Background()
err = commandEnv.MasterClient.WithClient(ctx, func(client master_pb.SeaweedClient) error {
2019-05-06 04:17:23 +00:00
resp, err = client.VolumeList(ctx, &master_pb.VolumeListRequest{})
return err
})
if err != nil {
return err
}
typeToNodes := collectVolumeServersByType(resp.TopologyInfo, *dc)
2019-12-24 01:58:47 +00:00
for maxVolumeCount, volumeServers := range typeToNodes {
2019-05-06 04:17:23 +00:00
if len(volumeServers) < 2 {
2019-12-24 01:58:47 +00:00
fmt.Printf("only 1 node is configured max %d volumes, skipping balancing\n", maxVolumeCount)
2019-05-06 04:17:23 +00:00
continue
}
if *collection == "EACH_COLLECTION" {
2019-05-30 16:27:23 +00:00
collections, err := ListCollectionNames(commandEnv, true, false)
if err != nil {
return err
}
for _, c := range collections {
if err = balanceVolumeServers(commandEnv, volumeServers, resp.VolumeSizeLimitMb*1024*1024, c, *applyBalancing); err != nil {
return err
}
}
2019-12-24 01:58:47 +00:00
} else if *collection == "ALL_COLLECTIONS" {
if err = balanceVolumeServers(commandEnv, volumeServers, resp.VolumeSizeLimitMb*1024*1024, "ALL_COLLECTIONS", *applyBalancing); err != nil {
return err
}
} else {
if err = balanceVolumeServers(commandEnv, volumeServers, resp.VolumeSizeLimitMb*1024*1024, *collection, *applyBalancing); err != nil {
return err
}
2019-05-06 04:58:46 +00:00
}
2019-05-06 04:17:23 +00:00
}
return nil
}
func balanceVolumeServers(commandEnv *CommandEnv, dataNodeInfos []*master_pb.DataNodeInfo, volumeSizeLimit uint64, collection string, applyBalancing bool) error {
2019-12-24 01:58:47 +00:00
2019-05-06 04:17:23 +00:00
var nodes []*Node
for _, dn := range dataNodeInfos {
nodes = append(nodes, &Node{
info: dn,
})
}
// balance writable volumes
for _, n := range nodes {
n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
2019-12-24 01:58:47 +00:00
if collection != "ALL_COLLECTIONS" {
2019-05-06 04:17:23 +00:00
if v.Collection != collection {
return false
}
}
return !v.ReadOnly && v.Size < volumeSizeLimit
})
}
2019-05-06 04:58:46 +00:00
if err := balanceSelectedVolume(commandEnv, nodes, sortWritableVolumes, applyBalancing); err != nil {
return err
}
2019-05-06 04:17:23 +00:00
// balance readable volumes
for _, n := range nodes {
n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool {
2019-12-24 01:58:47 +00:00
if collection != "ALL_COLLECTIONS" {
2019-05-06 04:17:23 +00:00
if v.Collection != collection {
return false
}
}
return v.ReadOnly || v.Size >= volumeSizeLimit
})
}
2019-05-06 04:58:46 +00:00
if err := balanceSelectedVolume(commandEnv, nodes, sortReadOnlyVolumes, applyBalancing); err != nil {
return err
}
return nil
2019-05-06 04:17:23 +00:00
}
func collectVolumeServersByType(t *master_pb.TopologyInfo, selectedDataCenter string) (typeToNodes map[uint64][]*master_pb.DataNodeInfo) {
2019-05-06 04:17:23 +00:00
typeToNodes = make(map[uint64][]*master_pb.DataNodeInfo)
for _, dc := range t.DataCenterInfos {
if selectedDataCenter != "" && dc.Id != selectedDataCenter {
continue
}
2019-05-06 04:17:23 +00:00
for _, r := range dc.RackInfos {
for _, dn := range r.DataNodeInfos {
typeToNodes[dn.MaxVolumeCount] = append(typeToNodes[dn.MaxVolumeCount], dn)
}
}
}
return
}
type Node struct {
info *master_pb.DataNodeInfo
selectedVolumes map[uint32]*master_pb.VolumeInformationMessage
}
func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) {
sort.Slice(volumes, func(i, j int) bool {
return volumes[i].Size < volumes[j].Size
})
}
func sortReadOnlyVolumes(volumes []*master_pb.VolumeInformationMessage) {
sort.Slice(volumes, func(i, j int) bool {
return volumes[i].Id < volumes[j].Id
})
}
func balanceSelectedVolume(commandEnv *CommandEnv, nodes []*Node, sortCandidatesFn func(volumes []*master_pb.VolumeInformationMessage), applyBalancing bool) error {
2019-05-06 04:17:23 +00:00
selectedVolumeCount := 0
for _, dn := range nodes {
selectedVolumeCount += len(dn.selectedVolumes)
}
2019-06-11 04:32:56 +00:00
idealSelectedVolumes := ceilDivide(selectedVolumeCount, len(nodes))
2019-05-06 04:17:23 +00:00
hasMove := true
for hasMove {
hasMove = false
sort.Slice(nodes, func(i, j int) bool {
2019-06-11 04:32:56 +00:00
// TODO sort by free volume slots???
2019-05-06 04:17:23 +00:00
return len(nodes[i].selectedVolumes) < len(nodes[j].selectedVolumes)
})
emptyNode, fullNode := nodes[0], nodes[len(nodes)-1]
if len(fullNode.selectedVolumes) > idealSelectedVolumes && len(emptyNode.selectedVolumes)+1 <= idealSelectedVolumes {
// sort the volumes to move
var candidateVolumes []*master_pb.VolumeInformationMessage
for _, v := range fullNode.selectedVolumes {
candidateVolumes = append(candidateVolumes, v)
}
sortCandidatesFn(candidateVolumes)
for _, v := range candidateVolumes {
if _, found := emptyNode.selectedVolumes[v.Id]; !found {
2019-05-06 04:58:46 +00:00
if err := moveVolume(commandEnv, v, fullNode, emptyNode, applyBalancing); err == nil {
delete(fullNode.selectedVolumes, v.Id)
emptyNode.selectedVolumes[v.Id] = v
hasMove = true
break
} else {
return err
}
2019-05-06 04:17:23 +00:00
}
}
}
}
2019-05-06 04:58:46 +00:00
return nil
2019-05-06 04:17:23 +00:00
}
func moveVolume(commandEnv *CommandEnv, v *master_pb.VolumeInformationMessage, fullNode *Node, emptyNode *Node, applyBalancing bool) error {
2019-05-06 04:17:23 +00:00
collectionPrefix := v.Collection + "_"
if v.Collection == "" {
collectionPrefix = ""
}
fmt.Fprintf(os.Stdout, "moving volume %s%d %s => %s\n", collectionPrefix, v.Id, fullNode.info.Id, emptyNode.info.Id)
if applyBalancing {
ctx := context.Background()
2019-05-06 04:58:46 +00:00
return LiveMoveVolume(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), fullNode.info.Id, emptyNode.info.Id, 5*time.Second)
2019-05-06 04:17:23 +00:00
}
2019-05-06 04:58:46 +00:00
return nil
2019-05-06 04:17:23 +00:00
}
func (node *Node) selectVolumes(fn func(v *master_pb.VolumeInformationMessage) bool) {
2019-05-06 04:17:23 +00:00
node.selectedVolumes = make(map[uint32]*master_pb.VolumeInformationMessage)
for _, v := range node.info.VolumeInfos {
if fn(v) {
node.selectedVolumes[v.Id] = v
}
}
}