From 3551ca2fcf423464afb2db4b5792c22ec94c2bfd Mon Sep 17 00:00:00 2001 From: justin Date: Mon, 18 Apr 2022 10:35:43 +0800 Subject: [PATCH 1/2] enhancement: replace sort.Slice with slices.SortFunc to reduce reflection --- go.mod | 5 +++-- weed/command/master.go | 6 +++--- weed/filer/filechunks.go | 18 ++++++++---------- weed/filer/filechunks2_test.go | 10 +++++----- weed/filer/filechunks_read.go | 17 +++++++---------- weed/filer/redis/universal_redis_store.go | 6 +++--- weed/filer/stream.go | 13 +++++++------ weed/mount/filehandle.go | 6 +++--- weed/s3api/filer_multipart.go | 5 +++-- weed/s3api/s3api_object_handlers.go | 6 +++--- .../filer_server_handlers_write_upload.go | 8 +++----- weed/shell/command_ec_balance.go | 15 +++++++-------- weed/shell/command_ec_common.go | 17 ++++++++--------- weed/shell/command_fs_meta_cat.go | 17 +++++++---------- weed/shell/command_volume_balance.go | 15 +++++++-------- weed/shell/command_volume_check_disk.go | 6 +++--- weed/shell/command_volume_fix_replication.go | 10 ++++------ weed/shell/command_volume_list.go | 18 +++++++++--------- weed/shell/command_volume_server_evacuate.go | 13 +++++-------- weed/shell/shell_liner.go | 8 +++----- weed/storage/disk_location_ec.go | 8 +++----- weed/storage/erasure_coding/ec_volume.go | 7 +++---- weed/storage/store_ec.go | 6 +++--- weed/util/chunk_cache/on_disk_cache_layer.go | 10 ++++------ weed/util/skiplist/name_batch.go | 6 +++--- 25 files changed, 117 insertions(+), 139 deletions(-) diff --git a/go.mod b/go.mod index 47f62b78f..10eb041e2 100644 --- a/go.mod +++ b/go.mod @@ -122,12 +122,13 @@ require ( gocloud.dev/pubsub/natspubsub v0.25.0 gocloud.dev/pubsub/rabbitpubsub v0.25.0 golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect + golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd golang.org/x/image v0.0.0-20200119044424-58c23975cae1 golang.org/x/net v0.0.0-20220401154927-543a649e0bdd golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 + golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/api v0.74.0 google.golang.org/appengine v1.6.7 // indirect @@ -194,7 +195,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect - golang.org/x/mod v0.5.0 // indirect + golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/weed/command/master.go b/weed/command/master.go index 3d69f4216..0d7fb882b 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -1,9 +1,9 @@ package command import ( + "golang.org/x/exp/slices" "net/http" "os" - "sort" "strings" "time" @@ -251,8 +251,8 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin } func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool { - sort.Slice(peers, func(i, j int) bool { - return strings.Compare(string(peers[i]), string(peers[j])) < 0 + slices.SortFunc(peers, func(a, b pb.ServerAddress) bool { + return strings.Compare(string(a), string(b)) < 0 }) if len(peers) <= 0 { return true diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go index 11a779147..208ef8095 100644 --- a/weed/filer/filechunks.go +++ b/weed/filer/filechunks.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" "github.com/chrislusf/seaweedfs/weed/wdclient" + "golang.org/x/exp/slices" "math" - "sort" "sync" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -254,19 +254,17 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction if true { return visibles2, err } - - sort.Slice(chunks, func(i, j int) bool { - if chunks[i].Mtime == chunks[j].Mtime { - filer_pb.EnsureFid(chunks[i]) - filer_pb.EnsureFid(chunks[j]) - if chunks[i].Fid == nil || chunks[j].Fid == nil { + slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool { + if a.Mtime == b.Mtime { + filer_pb.EnsureFid(a) + filer_pb.EnsureFid(b) + if a.Fid == nil || b.Fid == nil { return true } - return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey + return a.Fid.FileKey < b.Fid.FileKey } - return chunks[i].Mtime < chunks[j].Mtime // keep this to make tests run + return a.Mtime < b.Mtime }) - for _, chunk := range chunks { // glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) diff --git a/weed/filer/filechunks2_test.go b/weed/filer/filechunks2_test.go index 9f9566d9b..39dec87c9 100644 --- a/weed/filer/filechunks2_test.go +++ b/weed/filer/filechunks2_test.go @@ -1,7 +1,7 @@ package filer import ( - "sort" + "golang.org/x/exp/slices" "testing" "github.com/chrislusf/seaweedfs/weed/glog" @@ -34,11 +34,11 @@ func TestCompactFileChunksRealCase(t *testing.T) { } func printChunks(name string, chunks []*filer_pb.FileChunk) { - sort.Slice(chunks, func(i, j int) bool { - if chunks[i].Offset == chunks[j].Offset { - return chunks[i].Mtime < chunks[j].Mtime + slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool { + if a.Offset == b.Offset { + return a.Mtime < b.Mtime } - return chunks[i].Offset < chunks[j].Offset + return a.Offset < b.Offset }) for _, chunk := range chunks { glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) diff --git a/weed/filer/filechunks_read.go b/weed/filer/filechunks_read.go index 33ee6d138..1d0bd837a 100644 --- a/weed/filer/filechunks_read.go +++ b/weed/filer/filechunks_read.go @@ -2,7 +2,7 @@ package filer import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "sort" + "golang.org/x/exp/slices" ) func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) { @@ -22,17 +22,14 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterva isStart: false, }) } - sort.Slice(points, func(i, j int) bool { - if points[i].x != points[j].x { - return points[i].x < points[j].x + slices.SortFunc(points, func(a, b *Point) bool { + if a.x != b.x { + return a.x < b.x } - if points[i].ts != points[j].ts { - return points[i].ts < points[j].ts + if a.ts != b.ts { + return a.ts < b.ts } - if !points[i].isStart { - return true - } - return false + return !a.isStart }) var prevX int64 diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index 2cc7f49b1..0cdf58d7f 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -3,7 +3,7 @@ package redis import ( "context" "fmt" - "sort" + "golang.org/x/exp/slices" "strings" "time" @@ -157,8 +157,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP } // sort - sort.Slice(members, func(i, j int) bool { - return strings.Compare(members[i], members[j]) < 0 + slices.SortFunc(members, func(a, b string) bool { + return strings.Compare(a, b) < 0 }) // limit diff --git a/weed/filer/stream.go b/weed/filer/stream.go index 36278f0b1..7da9fd0a0 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -3,6 +3,7 @@ package filer import ( "bytes" "fmt" + "golang.org/x/exp/slices" "io" "math" "sort" @@ -39,11 +40,11 @@ func isSameChunks(a, b []*filer_pb.FileChunk) bool { if len(a) != len(b) { return false } - sort.Slice(a, func(i, j int) bool { - return strings.Compare(a[i].ETag, a[j].ETag) < 0 + slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool { + return strings.Compare(i.ETag, j.ETag) < 0 }) - sort.Slice(b, func(i, j int) bool { - return strings.Compare(b[i].ETag, b[j].ETag) < 0 + slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool { + return strings.Compare(i.ETag, j.ETag) < 0 }) for i := 0; i < len(a); i++ { if a[i].ETag != b[i].ETag { @@ -179,8 +180,8 @@ var _ = io.ReaderAt(&ChunkStreamReader{}) func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader { chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64) - sort.Slice(chunkViews, func(i, j int) bool { - return chunkViews[i].LogicOffset < chunkViews[j].LogicOffset + slices.SortFunc(chunkViews, func(a, b *ChunkView) bool { + return a.LogicOffset < b.LogicOffset }) var totalSize int64 diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go index 86fbabfa7..8ce31a078 100644 --- a/weed/mount/filehandle.go +++ b/weed/mount/filehandle.go @@ -5,8 +5,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" + "golang.org/x/exp/slices" "io" - "sort" "sync" ) @@ -76,8 +76,8 @@ func (fh *FileHandle) addChunks(chunks []*filer_pb.FileChunk) { } // sort incoming chunks - sort.Slice(chunks, func(i, j int) bool { - return lessThan(chunks[i], chunks[j]) + slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool { + return lessThan(a, b) }) glog.V(4).Infof("%s existing %d chunks adds %d more", fh.FullPath(), len(fh.entry.Chunks), len(chunks)) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 64ce16b45..10a43a5d5 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "golang.org/x/exp/slices" "path/filepath" "sort" "strconv" @@ -69,8 +70,8 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa glog.V(2).Infof("completeMultipartUpload input %v", input) completedParts := parts.Parts - sort.Slice(completedParts, func(i, j int) bool { - return completedParts[i].PartNumber < completedParts[j].PartNumber + slices.SortFunc(completedParts, func(a, b CompletedPart) bool { + return a.PartNumber < b.PartNumber }) uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 190045517..3d26d395e 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -8,10 +8,10 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util/mem" + "golang.org/x/exp/slices" "io" "net/http" "net/url" - "sort" "strings" "time" @@ -290,8 +290,8 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli for dir, _ := range directoriesWithDeletion { allDirs = append(allDirs, dir) } - sort.Slice(allDirs, func(i, j int) bool { - return len(allDirs[i]) > len(allDirs[j]) + slices.SortFunc(allDirs, func(a, b string) bool { + return len(a) > len(b) }) newDirectoriesWithDeletion = make(map[string]int) for _, dir := range allDirs { diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index 6ee378819..fe3346402 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -4,10 +4,10 @@ import ( "bytes" "crypto/md5" "fmt" + "golang.org/x/exp/slices" "hash" "io" "net/http" - "sort" "strconv" "strings" "sync" @@ -130,11 +130,9 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque fs.filer.DeleteChunks(fileChunks) return nil, md5Hash, 0, uploadErr, nil } - - sort.Slice(fileChunks, func(i, j int) bool { - return fileChunks[i].Offset < fileChunks[j].Offset + slices.SortFunc(fileChunks, func(a, b *filer_pb.FileChunk) bool { + return a.Offset < b.Offset }) - return fileChunks, md5Hash, chunkOffset, nil, smallContent } diff --git a/weed/shell/command_ec_balance.go b/weed/shell/command_ec_balance.go index 6cd91119b..393d44b80 100644 --- a/weed/shell/command_ec_balance.go +++ b/weed/shell/command_ec_balance.go @@ -4,12 +4,11 @@ import ( "flag" "fmt" "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/storage/types" - "io" - "sort" - "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" + "io" ) func init() { @@ -411,8 +410,8 @@ func doBalanceEcRack(commandEnv *CommandEnv, ecRack *EcRack, applyBalancing bool hasMove := true for hasMove { hasMove = false - sort.Slice(rackEcNodes, func(i, j int) bool { - return rackEcNodes[i].freeEcSlot > rackEcNodes[j].freeEcSlot + slices.SortFunc(rackEcNodes, func(a, b *EcNode) bool { + return a.freeEcSlot > b.freeEcSlot }) emptyNode, fullNode := rackEcNodes[0], rackEcNodes[len(rackEcNodes)-1] emptyNodeShardCount, fullNodeShardCount := ecNodeIdToShardCount[emptyNode.info.Id], ecNodeIdToShardCount[fullNode.info.Id] @@ -492,8 +491,8 @@ func pickNEcShardsToMoveFrom(ecNodes []*EcNode, vid needle.VolumeId, n int) map[ }) } } - sort.Slice(candidateEcNodes, func(i, j int) bool { - return candidateEcNodes[i].shardCount > candidateEcNodes[j].shardCount + slices.SortFunc(candidateEcNodes, func(a, b *CandidateEcNode) bool { + return a.shardCount > b.shardCount }) for i := 0; i < n; i++ { selectedEcNodeIndex := -1 diff --git a/weed/shell/command_ec_common.go b/weed/shell/command_ec_common.go index b3bd0ce5d..27b650731 100644 --- a/weed/shell/command_ec_common.go +++ b/weed/shell/command_ec_common.go @@ -3,18 +3,17 @@ package shell import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/storage/types" - "math" - "sort" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "google.golang.org/grpc" + "math" ) func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, applyBalancing bool) (err error) { @@ -116,14 +115,14 @@ func eachDataNode(topo *master_pb.TopologyInfo, fn func(dc string, rack RackId, } func sortEcNodesByFreeslotsDecending(ecNodes []*EcNode) { - sort.Slice(ecNodes, func(i, j int) bool { - return ecNodes[i].freeEcSlot > ecNodes[j].freeEcSlot + slices.SortFunc(ecNodes, func(a, b *EcNode) bool { + return a.freeEcSlot > b.freeEcSlot }) } func sortEcNodesByFreeslotsAscending(ecNodes []*EcNode) { - sort.Slice(ecNodes, func(i, j int) bool { - return ecNodes[i].freeEcSlot < ecNodes[j].freeEcSlot + slices.SortFunc(ecNodes, func(a, b *EcNode) bool { + return a.freeEcSlot < b.freeEcSlot }) } diff --git a/weed/shell/command_fs_meta_cat.go b/weed/shell/command_fs_meta_cat.go index a7de6d3ef..4616c072d 100644 --- a/weed/shell/command_fs_meta_cat.go +++ b/weed/shell/command_fs_meta_cat.go @@ -2,11 +2,10 @@ package shell import ( "fmt" - "github.com/golang/protobuf/proto" - "io" - "sort" - "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/proto" + "golang.org/x/exp/slices" + "io" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -55,14 +54,12 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W EmitDefaults: true, Indent: " ", } - - sort.Slice(respLookupEntry.Entry.Chunks, func(i, j int) bool { - if respLookupEntry.Entry.Chunks[i].Offset == respLookupEntry.Entry.Chunks[j].Offset { - return respLookupEntry.Entry.Chunks[i].Mtime < respLookupEntry.Entry.Chunks[j].Mtime + slices.SortFunc(respLookupEntry.Entry.Chunks, func(a, b *filer_pb.FileChunk) bool { + if a.Offset == b.Offset { + return a.Mtime < b.Mtime } - return respLookupEntry.Entry.Chunks[i].Offset < respLookupEntry.Entry.Chunks[j].Offset + return a.Offset < b.Offset }) - text, marshalErr := m.MarshalToString(respLookupEntry.Entry) if marshalErr != nil { return fmt.Errorf("marshal meta: %v", marshalErr) diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go index 7a983de1a..b01d348c5 100644 --- a/weed/shell/command_volume_balance.go +++ b/weed/shell/command_volume_balance.go @@ -6,9 +6,9 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "io" "os" - "sort" "time" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -224,14 +224,14 @@ func (n *Node) selectVolumes(fn func(v *master_pb.VolumeInformationMessage) bool } func sortWritableVolumes(volumes []*master_pb.VolumeInformationMessage) { - sort.Slice(volumes, func(i, j int) bool { - return volumes[i].Size < volumes[j].Size + slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool { + return a.Size < b.Size }) } func sortReadOnlyVolumes(volumes []*master_pb.VolumeInformationMessage) { - sort.Slice(volumes, func(i, j int) bool { - return volumes[i].Id < volumes[j].Id + slices.SortFunc(volumes, func(a, b *master_pb.VolumeInformationMessage) bool { + return a.Id < b.Id }) } @@ -255,10 +255,9 @@ func balanceSelectedVolume(commandEnv *CommandEnv, diskType types.DiskType, volu for hasMoved { hasMoved = false - sort.Slice(nodesWithCapacity, func(i, j int) bool { - return nodesWithCapacity[i].localVolumeRatio(capacityFunc) < nodesWithCapacity[j].localVolumeRatio(capacityFunc) + slices.SortFunc(nodesWithCapacity, func(a, b *Node) bool { + return a.localVolumeRatio(capacityFunc) < b.localVolumeRatio(capacityFunc) }) - fullNode := nodesWithCapacity[len(nodesWithCapacity)-1] var candidateVolumes []*master_pb.VolumeInformationMessage for _, v := range fullNode.selectedVolumes { diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index 54edd53dd..53284096d 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -9,9 +9,9 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/storage/needle_map" + "golang.org/x/exp/slices" "io" "math" - "sort" ) func init() { @@ -70,8 +70,8 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write } for _, replicas := range volumeReplicas { - sort.Slice(replicas, func(i, j int) bool { - return fileCount(replicas[i]) > fileCount(replicas[j]) + slices.SortFunc(replicas, func(a, b *VolumeReplica) bool { + return fileCount(a) > fileCount(b) }) for len(replicas) >= 2 { a, b := replicas[0], replicas[1] diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 78285d8a5..c4bef5925 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -7,9 +7,9 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "io" "path/filepath" - "sort" "strconv" "time" @@ -308,8 +308,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co func keepDataNodesSorted(dataNodes []location, diskType types.DiskType) { fn := capacityByFreeVolumeCount(diskType) - sort.Slice(dataNodes, func(i, j int) bool { - return fn(dataNodes[i].dataNode) > fn(dataNodes[j].dataNode) + slices.SortFunc(dataNodes, func(a, b location) bool { + return fn(a.dataNode) > fn(b.dataNode) }) } @@ -488,9 +488,7 @@ func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[st } func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica { - - sort.Slice(replicas, func(i, j int) bool { - a, b := replicas[i], replicas[j] + slices.SortFunc(replicas, func(a, b *VolumeReplica) bool { if a.info.Size != b.info.Size { return a.info.Size < b.info.Size } diff --git a/weed/shell/command_volume_list.go b/weed/shell/command_volume_list.go index 4c0429ecb..9150752d5 100644 --- a/weed/shell/command_volume_list.go +++ b/weed/shell/command_volume_list.go @@ -6,9 +6,9 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" + "golang.org/x/exp/slices" "io" - "sort" ) func init() { @@ -67,8 +67,8 @@ func diskInfoToString(diskInfo *master_pb.DiskInfo) string { func writeTopologyInfo(writer io.Writer, t *master_pb.TopologyInfo, volumeSizeLimitMb uint64, verbosityLevel int) statistics { output(verbosityLevel >= 0, writer, "Topology volumeSizeLimit:%d MB%s\n", volumeSizeLimitMb, diskInfosToString(t.DiskInfos)) - sort.Slice(t.DataCenterInfos, func(i, j int) bool { - return t.DataCenterInfos[i].Id < t.DataCenterInfos[j].Id + slices.SortFunc(t.DataCenterInfos, func(a, b *master_pb.DataCenterInfo) bool { + return a.Id < b.Id }) var s statistics for _, dc := range t.DataCenterInfos { @@ -80,8 +80,8 @@ func writeTopologyInfo(writer io.Writer, t *master_pb.TopologyInfo, volumeSizeLi func writeDataCenterInfo(writer io.Writer, t *master_pb.DataCenterInfo, verbosityLevel int) statistics { output(verbosityLevel >= 1, writer, " DataCenter %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics - sort.Slice(t.RackInfos, func(i, j int) bool { - return t.RackInfos[i].Id < t.RackInfos[j].Id + slices.SortFunc(t.RackInfos, func(a, b *master_pb.RackInfo) bool { + return a.Id < b.Id }) for _, r := range t.RackInfos { s = s.plus(writeRackInfo(writer, r, verbosityLevel)) @@ -92,8 +92,8 @@ func writeDataCenterInfo(writer io.Writer, t *master_pb.DataCenterInfo, verbosit func writeRackInfo(writer io.Writer, t *master_pb.RackInfo, verbosityLevel int) statistics { output(verbosityLevel >= 2, writer, " Rack %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics - sort.Slice(t.DataNodeInfos, func(i, j int) bool { - return t.DataNodeInfos[i].Id < t.DataNodeInfos[j].Id + slices.SortFunc(t.DataNodeInfos, func(a, b *master_pb.DataNodeInfo) bool { + return a.Id < b.Id }) for _, dn := range t.DataNodeInfos { s = s.plus(writeDataNodeInfo(writer, dn, verbosityLevel)) @@ -118,8 +118,8 @@ func writeDiskInfo(writer io.Writer, t *master_pb.DiskInfo, verbosityLevel int) diskType = "hdd" } output(verbosityLevel >= 4, writer, " Disk %s(%s)\n", diskType, diskInfoToString(t)) - sort.Slice(t.VolumeInfos, func(i, j int) bool { - return t.VolumeInfos[i].Id < t.VolumeInfos[j].Id + slices.SortFunc(t.VolumeInfos, func(a, b *master_pb.VolumeInformationMessage) bool { + return a.Id < b.Id }) for _, vi := range t.VolumeInfos { s = s.plus(writeVolumeInformationMessage(writer, vi, verbosityLevel)) diff --git a/weed/shell/command_volume_server_evacuate.go b/weed/shell/command_volume_server_evacuate.go index 31ebcfec1..f07ea4b79 100644 --- a/weed/shell/command_volume_server_evacuate.go +++ b/weed/shell/command_volume_server_evacuate.go @@ -8,9 +8,9 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" "io" "os" - "sort" ) func init() { @@ -153,11 +153,9 @@ func evacuateEcVolumes(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyI func moveAwayOneEcVolume(commandEnv *CommandEnv, ecShardInfo *master_pb.VolumeEcShardInformationMessage, thisNode *EcNode, otherNodes []*EcNode, applyChange bool) (hasMoved bool, err error) { for _, shardId := range erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIds() { - - sort.Slice(otherNodes, func(i, j int) bool { - return otherNodes[i].localShardIdCount(ecShardInfo.Id) < otherNodes[j].localShardIdCount(ecShardInfo.Id) + slices.SortFunc(otherNodes, func(a, b *EcNode) bool { + return a.localShardIdCount(ecShardInfo.Id) < b.localShardIdCount(ecShardInfo.Id) }) - for i := 0; i < len(otherNodes); i++ { emptyNode := otherNodes[i] collectionPrefix := "" @@ -188,10 +186,9 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][ return v.DiskType == vol.DiskType }) } - sort.Slice(otherNodes, func(i, j int) bool { - return otherNodes[i].localVolumeRatio(fn) > otherNodes[j].localVolumeRatio(fn) + slices.SortFunc(otherNodes, func(a, b *Node) bool { + return a.localVolumeRatio(fn) > b.localVolumeRatio(fn) }) - for i := 0; i < len(otherNodes); i++ { emptyNode := otherNodes[i] hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, thisNode, vol, emptyNode, applyChange) diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index 90ce2dbb4..94a68f5bc 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -8,12 +8,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/util/grace" + "golang.org/x/exp/slices" "io" "math/rand" "os" "path" "regexp" - "sort" "strings" "github.com/peterh/liner" @@ -25,11 +25,9 @@ var ( ) func RunShell(options ShellOptions) { - - sort.Slice(Commands, func(i, j int) bool { - return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0 + slices.SortFunc(Commands, func(a, b command) bool { + return strings.Compare(a.Name(), b.Name()) < 0 }) - line = liner.NewLiner() defer line.Close() grace.OnInterrupt(func() { diff --git a/weed/storage/disk_location_ec.go b/weed/storage/disk_location_ec.go index 3f56d797b..5fa5316fd 100644 --- a/weed/storage/disk_location_ec.go +++ b/weed/storage/disk_location_ec.go @@ -2,10 +2,10 @@ package storage import ( "fmt" + "golang.org/x/exp/slices" "os" "path" "regexp" - "sort" "strconv" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" @@ -128,11 +128,9 @@ func (l *DiskLocation) loadAllEcShards() (err error) { } dirEntries = append(dirEntries, indexDirEntries...) } - - sort.Slice(dirEntries, func(i, j int) bool { - return dirEntries[i].Name() < dirEntries[j].Name() + slices.SortFunc(dirEntries, func(a, b os.DirEntry) bool { + return a.Name() < b.Name() }) - var sameVolumeShards []string var prevVolumeId needle.VolumeId for _, fileInfo := range dirEntries { diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index f4cde310f..4dd07ec40 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -5,9 +5,9 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/volume_info" + "golang.org/x/exp/slices" "math" "os" - "sort" "sync" "time" @@ -82,9 +82,8 @@ func (ev *EcVolume) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool { } } ev.Shards = append(ev.Shards, ecVolumeShard) - sort.Slice(ev.Shards, func(i, j int) bool { - return ev.Shards[i].VolumeId < ev.Shards[j].VolumeId || - ev.Shards[i].VolumeId == ev.Shards[j].VolumeId && ev.Shards[i].ShardId < ev.Shards[j].ShardId + slices.SortFunc(ev.Shards, func(a, b *EcVolumeShard) bool { + return a.VolumeId < b.VolumeId || a.VolumeId == b.VolumeId && a.ShardId < b.ShardId }) return true } diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index 70e1593a0..0c9de45aa 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -4,9 +4,9 @@ import ( "context" "fmt" "github.com/chrislusf/seaweedfs/weed/pb" + "golang.org/x/exp/slices" "io" "os" - "sort" "sync" "time" @@ -389,8 +389,8 @@ func (s *Store) EcVolumes() (ecVolumes []*erasure_coding.EcVolume) { } location.ecVolumesLock.RUnlock() } - sort.Slice(ecVolumes, func(i, j int) bool { - return ecVolumes[i].VolumeId > ecVolumes[j].VolumeId + slices.SortFunc(ecVolumes, func(a, b *erasure_coding.EcVolume) bool { + return a.VolumeId > b.VolumeId }) return ecVolumes } diff --git a/weed/util/chunk_cache/on_disk_cache_layer.go b/weed/util/chunk_cache/on_disk_cache_layer.go index 9115b1bb1..de32fb445 100644 --- a/weed/util/chunk_cache/on_disk_cache_layer.go +++ b/weed/util/chunk_cache/on_disk_cache_layer.go @@ -2,12 +2,11 @@ package chunk_cache import ( "fmt" - "path" - "sort" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage/types" + "golang.org/x/exp/slices" + "path" ) type OnDiskCacheLayer struct { @@ -33,10 +32,9 @@ func NewOnDiskCacheLayer(dir, namePrefix string, diskSize int64, segmentCount in } // keep newest cache to the front - sort.Slice(c.diskCaches, func(i, j int) bool { - return c.diskCaches[i].lastModTime.After(c.diskCaches[j].lastModTime) + slices.SortFunc(c.diskCaches, func(a, b *ChunkCacheVolume) bool { + return a.lastModTime.After(b.lastModTime) }) - return c } diff --git a/weed/util/skiplist/name_batch.go b/weed/util/skiplist/name_batch.go index 71e5aeeba..53db5918f 100644 --- a/weed/util/skiplist/name_batch.go +++ b/weed/util/skiplist/name_batch.go @@ -3,7 +3,7 @@ package skiplist import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/golang/protobuf/proto" - "sort" + "golang.org/x/exp/slices" "strings" ) @@ -41,8 +41,8 @@ func (nb *NameBatch) ListNames(startFrom string, visitNamesFn func(name string) names = append(names, n) } } - sort.Slice(names, func(i, j int) bool { - return strings.Compare(names[i], names[j]) < 0 + slices.SortFunc(names, func(a, b string) bool { + return strings.Compare(a, b) < 0 }) for _, n := range names { if !visitNamesFn(n) { From 846c56e005243aca99dd4500b5680f340525a320 Mon Sep 17 00:00:00 2001 From: justin Date: Mon, 18 Apr 2022 10:35:59 +0800 Subject: [PATCH 2/2] enhancement: replace sort.Slice with slices.SortFunc to reduce reflection --- go.sum | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index cee2b8b40..fbcef93a0 100644 --- a/go.sum +++ b/go.sum @@ -965,6 +965,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd h1:zVFyTKZN/Q7mNRWSs1GOYnHM9NiFSJ54YVRsD0rNWT4= +golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -993,8 +995,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1266,8 +1269,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=