gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

working b2 sink

Browse source
This commit is contained in:
Chris Lu 2018-11-04 11:58:59 -08:00
parent f050b22d6c
commit 8dfac6a4cf
6 changed files with 55 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed/command/filer_replication.go
View file

@ -115,6 +115,8 @@ func runFilerReplicate(cmd *Command, args []string) bool {
}
if err = replicator.Replicate(key, m); err != nil {
glog.Errorf("replicate %s: %+v", key, err)
} else {
glog.V(4).Infof("replicated %s", key)
}
}

11
weed/command/scaffold.go
View file

@ -209,7 +209,7 @@ aws_access_key_id = "" # if empty, loads from the shared credentials fil
aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
region = "us-east-2"
bucket = "your_bucket_name" # an existing bucket
directory = "" # destination directory (do not prefix or suffix with "/")
directory = "/" # destination directory
[sink.google_cloud_storage]
# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
@ -224,15 +224,14 @@ enabled = false
account_name = ""
account_key = ""
container = "mycontainer" # an existing container
directory = "" # destination directory (do not prefix or suffix with "/")
directory = "/" # destination directory
[sink.backblaze]
# experimental, let me know if it works
enabled = false
account_id = ""
account_key = ""
b2_account_id = ""
b2_master_application_key = ""
bucket = "mybucket" # an existing bucket
directory = "" # destination directory (do not prefix or suffix with "/")
directory = "/" # destination directory
`
)

4
weed/replication/replicator.go
View file

@ -34,7 +34,9 @@ func (r *Replicator) Replicate(key string, message *filer_pb.EventNotification)
glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
return nil
}
key = filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):])
newKey := filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):])
glog.V(3).Infof("replicate %s => %s", key, newKey)
key = newKey
if message.OldEntry != nil && message.NewEntry == nil {
glog.V(4).Infof("deleting %v", key)
return r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks)

13
weed/replication/sink/azuresink/azure_sink.go
View file

@ -5,6 +5,7 @@ import (
"context"
"fmt"
"net/url"
"strings"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/chrislusf/seaweedfs/weed/filer2"
@ -71,6 +72,8 @@ func (g *AzureSink) initialize(accountName, accountKey, container, dir string) e
func (g *AzureSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
key = cleanKey(key)
if isDirectory {
key = key + "/"
}
@ -88,6 +91,8 @@ func (g *AzureSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks boo
func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
key = cleanKey(key)
if entry.IsDirectory {
return nil
}
@ -132,6 +137,14 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
}
func (g *AzureSink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
key = cleanKey(key)
// TODO improve efficiency
return false, nil
}
func cleanKey(key string) string {
if strings.HasPrefix(key, "/") {
key = key[1:]
}
return key
}

22
weed/replication/sink/b2sink/b2_sink.go
View file

@ -2,6 +2,8 @@ package B2Sink
import (
"context"
"strings"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication/sink"
@ -31,8 +33,8 @@ func (g *B2Sink) GetSinkToDirectory() string {
func (g *B2Sink) Initialize(configuration util.Configuration) error {
return g.initialize(
configuration.GetString("account_id"),
configuration.GetString("account_key"),
configuration.GetString("b2_account_id"),
configuration.GetString("b2_master_application_key"),
configuration.GetString("bucket"),
configuration.GetString("directory"),
)
@ -46,7 +48,7 @@ func (g *B2Sink) initialize(accountId, accountKey, bucket, dir string) error {
ctx := context.Background()
client, err := b2.NewClient(ctx, accountId, accountKey)
if err != nil {
return nil
return err
}
g.client = client
@ -58,6 +60,8 @@ func (g *B2Sink) initialize(accountId, accountKey, bucket, dir string) error {
func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
key = cleanKey(key)
if isDirectory {
key = key + "/"
}
@ -77,6 +81,8 @@ func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool)
func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
key = cleanKey(key)
if entry.IsDirectory {
return nil
}
@ -123,6 +129,16 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
}
func (g *B2Sink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
key = cleanKey(key)
// TODO improve efficiency
return false, nil
}
func cleanKey(key string) string {
if strings.HasPrefix(key, "/") {
key = key[1:]
}
return key
}

13
weed/replication/sink/s3sink/s3_sink.go
View file

@ -2,6 +2,7 @@ package S3Sink
import (
"fmt"
"strings"
"sync"
"github.com/aws/aws-sdk-go/aws"
@ -77,6 +78,8 @@ func (s3sink *S3Sink) initialize(awsAccessKeyId, aswSecretAccessKey, region, buc
func (s3sink *S3Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
key = cleanKey(key)
if isDirectory {
key = key + "/"
}
@ -87,6 +90,8 @@ func (s3sink *S3Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks b
func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
key = cleanKey(key)
if entry.IsDirectory {
return nil
}
@ -125,6 +130,14 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
}
func (s3sink *S3Sink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
key = cleanKey(key)
// TODO improve efficiency
return false, nil
}
func cleanKey(key string) string {
if strings.HasPrefix(key, "/") {
key = key[1:]
}
return key
}