This commit is contained in:
chrislu 2023-06-25 00:58:24 -07:00
commit ee4f7cd636

View file

@ -1,12 +1,14 @@
package rclone_backend
import (
"bytes"
"context"
"fmt"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
"os"
"text/template"
"time"
"github.com/google/uuid"
@ -38,15 +40,22 @@ func (factory *RcloneBackendFactory) BuildStorage(configuration backend.StringPr
}
type RcloneBackendStorage struct {
id string
remoteName string
fs fs.Fs
id string
remoteName string
keyTemplate *template.Template
keyTemplateText string
fs fs.Fs
}
func newRcloneBackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *RcloneBackendStorage, err error) {
s = &RcloneBackendStorage{}
s.id = id
s.remoteName = configuration.GetString(configPrefix + "remote_name")
s.keyTemplateText = configuration.GetString(configPrefix + "key_template")
s.keyTemplate, err = template.New("keyTemplate").Parse(s.keyTemplateText)
if err != nil {
return
}
ctx := context.TODO()
accounting.Start(ctx)
@ -65,9 +74,25 @@ func newRcloneBackendStorage(configuration backend.StringProperties, configPrefi
func (s *RcloneBackendStorage) ToProperties() map[string]string {
m := make(map[string]string)
m["remote_name"] = s.remoteName
if len(s.keyTemplateText) > 0 {
m["key_template"] = s.keyTemplateText
}
return m
}
func formatKey(key string, storage RcloneBackendStorage) (fKey string, err error) {
var b bytes.Buffer
if len(storage.keyTemplateText) == 0 {
fKey = key
} else {
err = storage.keyTemplate.Execute(&b, key)
if err == nil {
fKey = b.String()
}
}
return
}
func (s *RcloneBackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {
f := &RcloneBackendStorageFile{
backendStorage: s,
@ -85,6 +110,11 @@ func (s *RcloneBackendStorage) CopyFile(f *os.File, fn func(progressed int64, pe
}
key = randomUuid.String()
key, err = formatKey(key, *s)
if err != nil {
return key, 0, err
}
glog.V(1).Infof("copy dat file of %s to remote rclone.%s as %s", f.Name(), s.id, key)
util.Retry("upload via Rclone", func() error {