gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into mq-subscribe

Browse source
This commit is contained in:
chrislu 2023-12-28 11:57:30 -08:00
parent c950a40aad 183352c796
commit 9ed26cd7b0
6 changed files with 37 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -159,7 +159,7 @@ require (
github.com/shirou/gopsutil/v3 v3.23.11
github.com/tikv/client-go/v2 v2.0.7
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
github.com/ydb-platform/ydb-go-sdk/v3 v3.53.3
github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
)

4
go.sum
View file

@ -961,8 +961,8 @@ github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 h1:IG5bPd+Lqyc+zsw2kmxqfG
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0/go.mod h1:l6lZ+osdQOjDRBgRA4PQ06BuvmXN2neYjnRw8rCfd7s=
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
github.com/ydb-platform/ydb-go-sdk/v3 v3.53.3 h1:yv0RqgpJUtx32CJzU0se1fi+AJZ1V+F3wD4mP0ydkls=
github.com/ydb-platform/ydb-go-sdk/v3 v3.53.3/go.mod h1:qtQwB//TJJPH4YisuX/87ZNplf09eAP1FhrrnGJG6ME=
github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2 h1:E0yUuuX7UmPxXm92+yQCjMveLFO3zfvYFIJVuAqsVRA=
github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2/go.mod h1:fjBLQ2TdQNl4bMjuWl9adoTGBypwUTPoGC+EqYqiIcU=
github.com/ydb-platform/ydb-go-yc v0.10.2 h1:RAHy6g7ncxk1y0N4oS2MwYXLATqRqKBI6DYXuxpV2wo=
github.com/ydb-platform/ydb-go-yc v0.10.2/go.mod h1:U1dX3LJy6zADId2DciCXlgrU/vphK1+CQzaefKq21dQ=
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=

4
k8s/charts/seaweedfs/templates/master-servicemonitor.yaml
View file

@ -18,7 +18,7 @@ spec:
scrapeTimeout: 5s
selector:
matchLabels:
app: {{ template "seaweedfs.name" . }}
component: master
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
app.kubernetes.io/component: master
{{- end }}
{{- end }}

3
k8s/charts/seaweedfs/templates/master-statefulset.yaml
View file

@ -127,7 +127,8 @@ spec:
-metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
{{- end }}
{{- end }}
{{- else if .Values.master.metricsPort }}
{{- end }}
{{- if .Values.master.metricsPort }}
-metricsPort={{ .Values.master.metricsPort }} \
{{- end }}
-volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \

8
weed/server/volume_grpc_read_all.go
View file

@ -10,14 +10,14 @@ import (
func (vs *VolumeServer) ReadAllNeedles(req *volume_server_pb.ReadAllNeedlesRequest, stream volume_server_pb.VolumeServer_ReadAllNeedlesServer) (err error) {
for _, vid := range req.VolumeIds {
if err := vs.streamReadOneVolume(needle.VolumeId(vid), stream, err); err != nil {
if err := vs.streamReadOneVolume(needle.VolumeId(vid), stream); err != nil {
return err
}
}
return nil
}
func (vs *VolumeServer) streamReadOneVolume(vid needle.VolumeId, stream volume_server_pb.VolumeServer_ReadAllNeedlesServer, err error) error {
func (vs *VolumeServer) streamReadOneVolume(vid needle.VolumeId, stream volume_server_pb.VolumeServer_ReadAllNeedlesServer) error {
v := vs.store.GetVolume(vid)
if v == nil {
return fmt.Errorf("not found volume id %d", vid)
@ -30,7 +30,5 @@ func (vs *VolumeServer) streamReadOneVolume(vid needle.VolumeId, stream volume_s
offset := int64(v.SuperBlock.BlockSize())
err = storage.ScanVolumeFileFrom(v.Version(), v.DataBackend, offset, scanner)
return err
return storage.ScanVolumeFileFrom(v.Version(), v.DataBackend, offset, scanner)
}

32
weed/shell/command_fs_meta_load.go
View file

@ -6,6 +6,7 @@ import (
"io"
"os"
"strings"
"sync"
"time"
"google.golang.org/protobuf/proto"
@ -47,6 +48,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
metaLoadCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
c.dirPrefix = metaLoadCommand.String("dirPrefix", "", "load entries only with directories matching prefix")
concurrency := metaLoadCommand.Int("concurrency", 1, "number of parallel meta load to filer")
verbose := metaLoadCommand.Bool("v", true, "verbose mode")
if err = metaLoadCommand.Parse(args[0 : len(args)-1]); err != nil {
return nil
@ -64,6 +66,9 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
sizeBuf := make([]byte, 4)
waitChan := make(chan struct{}, *concurrency)
defer close(waitChan)
var wg sync.WaitGroup
for {
if n, err := dst.Read(sizeBuf); n != 4 {
@ -105,21 +110,34 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
}
fullEntry.Entry.Name = strings.ReplaceAll(fullEntry.Entry.Name, "/", "x")
if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
if fullEntry.Entry.IsDirectory {
wg.Wait()
if errEntry := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
Directory: fullEntry.Dir,
Entry: fullEntry.Entry,
}); err != nil {
return err
}); errEntry != nil {
return errEntry
}
if fullEntry.Entry.IsDirectory {
dirCount++
} else {
wg.Add(1)
waitChan <- struct{}{}
go func(entry *filer_pb.FullEntry) {
if errEntry := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
Directory: entry.Dir,
Entry: entry.Entry,
}); errEntry != nil {
err = errEntry
}
defer wg.Done()
<-waitChan
}(fullEntry)
if err != nil {
return err
}
fileCount++
}
}
})
if err == nil {