From cf552417a7a422d1313c53972fd1175684e758e0 Mon Sep 17 00:00:00 2001 From: bingoohuang Date: Tue, 27 Apr 2021 10:37:24 +0800 Subject: [PATCH] minFreeSpace refactored --- .../repeated_vacuum/repeated_vacuum.go | 2 +- weed/command/server.go | 9 +- weed/command/volume.go | 26 ++---- weed/server/volume_server.go | 2 +- weed/storage/disk_location.go | 14 ++- weed/storage/store.go | 4 +- weed/util/bytes.go | 22 ++--- weed/util/bytes_test.go | 26 ------ weed/util/minfreespace.go | 90 +++++++++++++++++++ weed/util/minfreespace_test.go | 29 ++++++ 10 files changed, 147 insertions(+), 77 deletions(-) create mode 100644 weed/util/minfreespace.go create mode 100644 weed/util/minfreespace_test.go diff --git a/unmaintained/repeated_vacuum/repeated_vacuum.go b/unmaintained/repeated_vacuum/repeated_vacuum.go index bff5becc1..d85e45af0 100644 --- a/unmaintained/repeated_vacuum/repeated_vacuum.go +++ b/unmaintained/repeated_vacuum/repeated_vacuum.go @@ -52,7 +52,7 @@ func main() { } func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) { - assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{ + assignResult, err := operation.Assign(func() string { return *master }, grpcDialOption, &operation.VolumeAssignRequest{ Count: 1, Replication: *replication, }) diff --git a/weed/command/server.go b/weed/command/server.go index 6eb3bf97c..d0020d33b 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -58,7 +58,8 @@ var ( serverDisableHttp = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.") volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.") - volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.") + volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).") + volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.") serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") // pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") @@ -244,8 +245,8 @@ func runServer(cmd *Command, args []string) bool { // start volume server if *isStartingVolumeServer { - go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent) - + minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent) + go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, minFreeSpaces) } if *isStartingMasterServer { @@ -253,6 +254,4 @@ func runServer(cmd *Command, args []string) bool { } select {} - - return true } diff --git a/weed/command/volume.go b/weed/command/volume.go index 93a19102d..139a3791e 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -57,7 +57,6 @@ type VolumeServerOptions struct { compactionMBPerSecond *int fileSizeLimitMB *int concurrentUploadLimitMB *int - minFreeSpaces []float32 pprof *bool preStopSeconds *int metricsHttpPort *int @@ -121,13 +120,13 @@ func runVolume(cmd *Command, args []string) bool { go stats_collect.StartMetricsServer(*v.metricsHttpPort) - v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, - util.EmptyTo(*minFreeSpace, *minFreeSpacePercent)) + minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent) + v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces) return true } -func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpace string) { +func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string, minFreeSpaces []util.MinFreeSpace) { // Set multiple folders and each folder's max volume count limit' v.folders = strings.Split(volumeFolders, ",") @@ -155,22 +154,13 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits)) } - // set minFreeSpacePercent - minFreeSpaceStrings := strings.Split(minFreeSpace, ",") - for _, freeString := range minFreeSpaceStrings { - if vv, e := util.ParseMinFreeSpace(freeString); e == nil { - v.minFreeSpaces = append(v.minFreeSpaces, vv) - } else { - glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString) - } - } - if len(v.minFreeSpaces) == 1 && len(v.folders) > 1 { + if len(minFreeSpaces) == 1 && len(v.folders) > 1 { for i := 0; i < len(v.folders)-1; i++ { - v.minFreeSpaces = append(v.minFreeSpaces, v.minFreeSpaces[0]) + minFreeSpaces = append(minFreeSpaces, minFreeSpaces[0]) } } - if len(v.folders) != len(v.minFreeSpaces) { - glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpaces)) + if len(v.folders) != len(minFreeSpaces) { + glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(minFreeSpaces)) } // set disk types @@ -233,7 +223,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux, *v.ip, *v.port, *v.publicUrl, - v.folders, v.folderMaxLimits, v.minFreeSpaces, diskTypes, + v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, *v.idxFolder, volumeNeedleMapKind, strings.Split(masters, ","), 5, *v.dataCenter, *v.rack, diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index f29f12148..f7359ea6b 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -43,7 +43,7 @@ type VolumeServer struct { func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, port int, publicUrl string, - folders []string, maxCounts []int, minFreeSpaces []float32, diskTypes []types.DiskType, + folders []string, maxCounts []int, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType, idxFolder string, needleMapKind storage.NeedleMapKind, masterNodes []string, pulseSeconds int, diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 3246718c8..33dd272ce 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -23,10 +23,9 @@ type DiskLocation struct { DiskType types.DiskType MaxVolumeCount int OriginalMaxVolumeCount int - // MinFreeSpace limits the minimum free space (<=100 as percentage, > 100 as bytes) - MinFreeSpace float32 - volumes map[needle.VolumeId]*Volume - volumesLock sync.RWMutex + MinFreeSpace util.MinFreeSpace + volumes map[needle.VolumeId]*Volume + volumesLock sync.RWMutex // erasure coding ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume @@ -35,7 +34,7 @@ type DiskLocation struct { isDiskSpaceLow bool } -func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace float32, idxDir string, diskType types.DiskType) *DiskLocation { +func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation { dir = util.ResolvePath(dir) if idxDir == "" { idxDir = dir @@ -363,7 +362,7 @@ func (l *DiskLocation) CheckDiskSpace() { stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used)) stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free)) - isLow := l.MinFreeSpace < 100 && s.PercentFree < l.MinFreeSpace || s.Free < uint64(l.MinFreeSpace) + isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree) if isLow != l.isDiskSpaceLow { l.isDiskSpaceLow = !l.isDiskSpaceLow } @@ -373,8 +372,7 @@ func (l *DiskLocation) CheckDiskSpace() { logLevel = glog.Level(0) } - glog.V(logLevel).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", - dir, s.PercentFree, l.MinFreeSpace, l.isDiskSpaceLow) + glog.V(logLevel).Infof("dir %s %s", dir, desc) } time.Sleep(time.Minute) } diff --git a/weed/storage/store.go b/weed/storage/store.go index 18496890c..f27f2412f 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/util" "path/filepath" "strings" "sync/atomic" @@ -52,7 +53,8 @@ func (s *Store) String() (str string) { return } -func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpaces []float32, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) { +func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, + minFreeSpaces []util.MinFreeSpace, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) { s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapKind: needleMapKind} s.Locations = make([]*DiskLocation, 0) for i := 0; i < len(dirnames); i++ { diff --git a/weed/util/bytes.go b/weed/util/bytes.go index 260e5067e..26da91033 100644 --- a/weed/util/bytes.go +++ b/weed/util/bytes.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "crypto/rand" "encoding/base64" - "errors" "fmt" "io" "math" @@ -176,23 +175,12 @@ func EmptyTo(s, to string) string { return s } -var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid") - -// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G -func ParseMinFreeSpace(s string) (float32, error) { - if value, e := strconv.ParseFloat(s, 32); e == nil { - if value < 0 || value > 100 { - return 0, ErrMinFreeSpaceBadValue - } - return float32(value), nil - } else if directSize, e2 := ParseBytes(s); e2 == nil { - if directSize <= 100 { - return 0, ErrMinFreeSpaceBadValue - } - return float32(directSize), nil +// IfElse works like b ? this : that. +func IfElse(b bool, this, that string) string { + if b { + return this } - - return 0, ErrMinFreeSpaceBadValue + return that } // ParseBytes parses a string representation of bytes into the number diff --git a/weed/util/bytes_test.go b/weed/util/bytes_test.go index 4a9c25e52..d9269cadb 100644 --- a/weed/util/bytes_test.go +++ b/weed/util/bytes_test.go @@ -2,32 +2,6 @@ package util import "testing" -func TestParseMinFreeSpace(t *testing.T) { - tests := []struct { - in string - ok bool - value float32 - }{ - {in: "42", ok: true, value: 42}, - {in: "-1", ok: false, value: 0}, - {in: "101", ok: false, value: 0}, - {in: "100B", ok: false, value: 0}, - {in: "100Ki", ok: true, value: 100 * 1024}, - {in: "100GiB", ok: true, value: 100 * 1024 * 1024 * 1024}, - {in: "42M", ok: true, value: 42 * 1000 * 1000}, - } - - for _, p := range tests { - got, err := ParseMinFreeSpace(p.in) - if p.ok != (err == nil) { - t.Errorf("failed to test %v", p.in) - } - if p.ok && err == nil && got != p.value { - t.Errorf("failed to test %v", p.in) - } - } -} - func TestByteParsing(t *testing.T) { tests := []struct { in string diff --git a/weed/util/minfreespace.go b/weed/util/minfreespace.go new file mode 100644 index 000000000..c802bf6dd --- /dev/null +++ b/weed/util/minfreespace.go @@ -0,0 +1,90 @@ +package util + +import ( + "errors" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "strconv" + "strings" +) + +// MinFreeSpaceType is the type of MinFreeSpace. +type MinFreeSpaceType int + +const ( + // AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100. + AsPercent MinFreeSpaceType = iota + // AsBytes set the MinFreeSpaceType to a absolute value bytes. + AsBytes +) + +// MinFreeSpace is type that defines the limit for the minimum free space. +type MinFreeSpace struct { + Type MinFreeSpaceType + Bytes uint64 + Percent float32 + Raw string +} + +// IsLow tells whether the free space is low or not. +func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string) { + switch s.Type { + case AsPercent: + yes = freePercent < s.Percent + op := IfElse(yes, "<", ">=") + return yes, fmt.Sprintf("disk free %.2f%% %s required %.2f%%", freePercent, op, s.Percent) + case AsBytes: + yes = freeBytes < s.Bytes + op := IfElse(yes, "<", ">=") + return yes, fmt.Sprintf("disk free %s %s required %s", + BytesToHumanReadable(freeBytes), op, BytesToHumanReadable(s.Bytes)) + } + + return false, "" +} + +// String returns a string representation of MinFreeSpace. +func (s MinFreeSpace) String() string { + switch s.Type { + case AsPercent: + return fmt.Sprintf("%.2f%%", s.Percent) + default: + return s.Raw + } +} + +// MustParseMinFreeSpace parses comma-separated argument for min free space setting. +// minFreeSpace has the high priority than minFreeSpacePercent if it is set. +func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace) { + ss := strings.Split(EmptyTo(minFreeSpace, minFreeSpacePercent), ",") + for _, freeString := range ss { + if vv, e := ParseMinFreeSpace(freeString); e == nil { + spaces = append(spaces, *vv) + } else { + glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString) + } + } + + return spaces +} + +var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid") + +// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G +func ParseMinFreeSpace(s string) (*MinFreeSpace, error) { + if percent, e := strconv.ParseFloat(s, 32); e == nil { + if percent < 0 || percent > 100 { + return nil, ErrMinFreeSpaceBadValue + } + return &MinFreeSpace{Type: AsPercent, Percent: float32(percent), Raw: s}, nil + } + + if directSize, e := ParseBytes(s); e == nil { + if directSize <= 100 { + return nil, ErrMinFreeSpaceBadValue + } + return &MinFreeSpace{Type: AsBytes, Bytes: directSize, Raw: s}, nil + } + + return nil, ErrMinFreeSpaceBadValue +} diff --git a/weed/util/minfreespace_test.go b/weed/util/minfreespace_test.go new file mode 100644 index 000000000..eec1942dd --- /dev/null +++ b/weed/util/minfreespace_test.go @@ -0,0 +1,29 @@ +package util + +import "testing" + +func TestParseMinFreeSpace(t *testing.T) { + tests := []struct { + in string + ok bool + value *MinFreeSpace + }{ + {in: "42", ok: true, value: &MinFreeSpace{Type: AsPercent, Percent: 42, Raw: "42"}}, + {in: "-1", ok: false, value: nil}, + {in: "101", ok: false, value: nil}, + {in: "100B", ok: false, value: nil}, + {in: "100Ki", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024, Raw: "100Ki"}}, + {in: "100GiB", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024 * 1024 * 1024, Raw: "100GiB"}}, + {in: "42M", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 42 * 1000 * 1000, Raw: "42M"}}, + } + + for _, p := range tests { + got, err := ParseMinFreeSpace(p.in) + if p.ok != (err == nil) { + t.Errorf("failed to test %v", p.in) + } + if p.ok && err == nil && *got != *p.value { + t.Errorf("failed to test %v", p.in) + } + } +}