gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

vacuum show LA

Browse source
This commit is contained in:
Konstantin Lebedev 2022-07-29 11:59:33 +05:00
parent 32d0413be1
commit 2f0dda384d
19 changed files with 914 additions and 879 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
weed/pb/filer_pb/filer.pb.go
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.21.4
// source: filer.proto
package filer_pb

1
weed/pb/filer_pb/filer_grpc.pb.go
View file

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedFilerClient is the client API for SeaweedFiler service.

4
weed/pb/iam_pb/iam.pb.go
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.21.4
// source: iam.proto
package iam_pb

1
weed/pb/iam_pb/iam_grpc.pb.go
View file

@ -8,7 +8,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service.

4
weed/pb/master_pb/master.pb.go
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.21.4
// source: master.proto
package master_pb

1
weed/pb/master_pb/master_grpc.pb.go
View file

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedClient is the client API for Seaweed service.

4
weed/pb/messaging_pb/messaging.pb.go
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.21.4
// source: messaging.proto
package messaging_pb

1
weed/pb/messaging_pb/messaging_grpc.pb.go
View file

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedMessagingClient is the client API for SeaweedMessaging service.

4
weed/pb/mount_pb/mount.pb.go
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.21.4
// source: mount.proto
package mount_pb

1
weed/pb/mount_pb/mount_grpc.pb.go
View file

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedMountClient is the client API for SeaweedMount service.

4
weed/pb/remote_pb/remote.pb.go
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc-gen-go v1.28.0
// protoc v3.21.4
// source: remote.proto
package remote_pb

2
weed/pb/s3_pb/s3.pb.go
View file

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc v3.21.1
// protoc v3.21.4
// source: s3.proto
package s3_pb

1
weed/pb/s3_pb/s3_grpc.pb.go
View file

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedS3Client is the client API for SeaweedS3 service.

1
weed/pb/volume_server.proto
View file

@ -147,6 +147,7 @@ message VacuumVolumeCompactRequest {
}
message VacuumVolumeCompactResponse {
int64 processed_bytes = 1;
uint32 percent_load_avg_1m = 2;
}
message VacuumVolumeCommitRequest {

1719
weed/pb/volume_server_pb/volume_server.pb.go
View file

File diff suppressed because it is too large Load diff

1
weed/pb/volume_server_pb/volume_server_grpc.pb.go
View file

@ -11,7 +11,6 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// VolumeServerClient is the client API for VolumeServer service.

22
weed/server/volume_grpc_vacuum.go
View file

@ -2,12 +2,17 @@ package weed_server
import (
"context"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/prometheus/procfs"
"github.com/prometheus/procfs/blockdevice"
"math"
"runtime"
)
var numCPU = runtime.NumCPU()
func (vs *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_server_pb.VacuumVolumeCheckRequest) (*volume_server_pb.VacuumVolumeCheckResponse, error) {
resp := &volume_server_pb.VacuumVolumeCheckResponse{}
@ -29,11 +34,24 @@ func (vs *VolumeServer) VacuumVolumeCompact(req *volume_server_pb.VacuumVolumeCo
resp := &volume_server_pb.VacuumVolumeCompactResponse{}
reportInterval := int64(1024 * 1024 * 128)
nextReportTarget := reportInterval
fs, fsErr := procfs.NewDefaultFS()
bd, bdErr := blockdevice.NewDefaultFS()
var sendErr error
err := vs.store.CompactVolume(needle.VolumeId(req.VolumeId), req.Preallocate, vs.compactionBytePerSecond, func(processed int64) bool {
if processed > nextReportTarget {
resp.ProcessedBytes = processed
if fsErr == nil && numCPU > 0 {
if fsLa, err := fs.LoadAvg(); err == nil {
resp.PercentLoadAvg_1M = uint32(math.Round(fsLa.Load1 * 100 / float64(numCPU)))
}
}
if bdErr == nil {
if bdStats, err := bd.ProcDiskstats(); err == nil {
for _, stat := range bdStats {
glog.V(0).Infof("vacuum id: processed ProcDiskstats: %+v", req.VolumeId, stat)
}
}
}
if sendErr = stream.Send(resp); sendErr != nil {
return false
}

10
weed/storage/volume_vacuum.go
View file

@ -88,7 +88,15 @@ func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64, prog
if err := v.nm.Sync(); err != nil {
glog.V(0).Infof("compact2 fail to sync volume idx %d: %v", v.Id, err)
}
return copyDataBasedOnIndexFile(v.FileName(".dat"), v.FileName(".idx"), v.FileName(".cpd"), v.FileName(".cpx"), v.SuperBlock, v.Version(), preallocate, compactionBytePerSecond, progressFn)
return copyDataBasedOnIndexFile(
v.FileName(".dat"), v.FileName(".idx"),
v.FileName(".cpd"), v.FileName(".cpx"),
v.SuperBlock,
v.Version(),
preallocate,
compactionBytePerSecond,
progressFn,
)
}
func (v *Volume) CommitCompact() error {

8
weed/topology/topology_vacuum.go
View file

@ -89,7 +89,8 @@ func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *
return recvErr
}
}
glog.V(0).Infof("%d vacuum %d on %s processed %d bytes", index, vid, url, resp.ProcessedBytes)
glog.V(0).Infof("%d vacuum %d on %s processed %d bytes, percentage loadAvg %v",
index, vid, url, resp.ProcessedBytes, resp.PercentLoadAvg_1M)
}
return nil
})
@ -138,6 +139,8 @@ func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *V
} else {
glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
}
// https://github.com/chrislusf/seaweedfs/issues/3369
time.Sleep(3 * time.Second)
}
if isCommitSuccess {
for _, dn := range locationlist.list {
@ -217,7 +220,8 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL
}
glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(
grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
if t.batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
t.batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
} else {