gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/seaweedfs/seaweedfs

Browse source
This commit is contained in:
chrislu 2023-06-25 00:58:24 -07:00
parent 3fd659df2a 0d9ba8c612
commit ee4f7cd636
1 changed files with 33 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
weed/storage/backend/rclone_backend/rclone_backend.go
View file

@ -1,12 +1,14 @@
package rclone_backend package rclone_backend
import ( import (
"bytes"
"context" "context"
"fmt" "fmt"
"github.com/rclone/rclone/fs/config/configfile" "github.com/rclone/rclone/fs/config/configfile"
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"io" "io"
"os" "os"
"text/template"
"time" "time"
"github.com/google/uuid" "github.com/google/uuid"
@ -38,15 +40,22 @@ func (factory *RcloneBackendFactory) BuildStorage(configuration backend.StringPr
} }
type RcloneBackendStorage struct { type RcloneBackendStorage struct {
id string id string
remoteName string remoteName string
fs fs.Fs keyTemplate *template.Template
keyTemplateText string
fs fs.Fs
} }
func newRcloneBackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *RcloneBackendStorage, err error) { func newRcloneBackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *RcloneBackendStorage, err error) {
s = &RcloneBackendStorage{} s = &RcloneBackendStorage{}
s.id = id s.id = id
s.remoteName = configuration.GetString(configPrefix + "remote_name") s.remoteName = configuration.GetString(configPrefix + "remote_name")
s.keyTemplateText = configuration.GetString(configPrefix + "key_template")
s.keyTemplate, err = template.New("keyTemplate").Parse(s.keyTemplateText)
if err != nil {
return
}
ctx := context.TODO() ctx := context.TODO()
accounting.Start(ctx) accounting.Start(ctx)
@ -65,9 +74,25 @@ func newRcloneBackendStorage(configuration backend.StringProperties, configPrefi
func (s *RcloneBackendStorage) ToProperties() map[string]string { func (s *RcloneBackendStorage) ToProperties() map[string]string {
m := make(map[string]string) m := make(map[string]string)
m["remote_name"] = s.remoteName m["remote_name"] = s.remoteName
if len(s.keyTemplateText) > 0 {
m["key_template"] = s.keyTemplateText
}
return m return m
} }
func formatKey(key string, storage RcloneBackendStorage) (fKey string, err error) {
var b bytes.Buffer
if len(storage.keyTemplateText) == 0 {
fKey = key
} else {
err = storage.keyTemplate.Execute(&b, key)
if err == nil {
fKey = b.String()
}
}
return
}
func (s *RcloneBackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile { func (s *RcloneBackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {
f := &RcloneBackendStorageFile{ f := &RcloneBackendStorageFile{
backendStorage: s, backendStorage: s,
@ -85,6 +110,11 @@ func (s *RcloneBackendStorage) CopyFile(f *os.File, fn func(progressed int64, pe
} }
key = randomUuid.String() key = randomUuid.String()
key, err = formatKey(key, *s)
if err != nil {
return key, 0, err
}
glog.V(1).Infof("copy dat file of %s to remote rclone.%s as %s", f.Name(), s.id, key) glog.V(1).Infof("copy dat file of %s to remote rclone.%s as %s", f.Name(), s.id, key)
util.Retry("upload via Rclone", func() error { util.Retry("upload via Rclone", func() error {