seaweedfs/weed/filer/arangodb/arangodb_store.go

348 lines
9.4 KiB
Go
Raw Normal View History

2022-03-17 09:49:26 +00:00
package arangodb
import (
"context"
"crypto/tls"
"fmt"
"strconv"
2022-03-18 02:12:25 +00:00
"strings"
"sync"
2022-03-17 09:49:26 +00:00
"time"
"github.com/arangodb/go-driver"
"github.com/arangodb/go-driver/http"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
func init() {
filer.Stores = append(filer.Stores, &ArangodbStore{})
}
var (
BUCKET_PREFIX = "/buckets"
DEFAULT_COLLECTION = "seaweed_no_bucket"
KVMETA_COLLECTION = "seaweed_kvmeta"
)
2022-03-17 09:49:26 +00:00
type ArangodbStore struct {
connect driver.Connection
client driver.Client
database driver.Database
kvCollection driver.Collection
buckets map[string]driver.Collection
mu sync.RWMutex
2022-03-18 02:12:25 +00:00
databaseName string
2022-03-17 09:49:26 +00:00
}
type Model struct {
2022-03-18 02:12:25 +00:00
Key string `json:"_key"`
Directory string `json:"directory,omitempty"`
Name string `json:"name,omitempty"`
2022-03-18 03:35:30 +00:00
Ttl string `json:"ttl,omitempty"`
2022-03-18 02:12:25 +00:00
//arangodb does not support binary blobs
//we encode byte slice into uint64 slice
//see helpers.go
Meta []uint64 `json:"meta"`
2022-03-17 09:49:26 +00:00
}
func (store *ArangodbStore) GetName() string {
return "arangodb"
}
func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
store.buckets = make(map[string]driver.Collection, 3)
2022-03-18 02:12:25 +00:00
store.databaseName = configuration.GetString(prefix + "db_name")
return store.connection(configuration.GetStringSlice(prefix+"servers"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
2022-03-18 02:12:25 +00:00
configuration.GetBool(prefix+"insecure_skip_verify"),
2022-03-17 09:49:26 +00:00
)
}
2022-03-18 02:12:25 +00:00
func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) {
2022-03-17 09:49:26 +00:00
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
store.connect, err = http.NewConnection(http.ConnectionConfig{
Endpoints: uris,
TLSConfig: &tls.Config{
2022-03-18 02:12:25 +00:00
InsecureSkipVerify: insecure,
2022-03-17 09:49:26 +00:00
},
})
if err != nil {
return err
}
store.client, err = driver.NewClient(driver.ClientConfig{
Connection: store.connect,
Authentication: driver.BasicAuthentication(user, pass),
})
if err != nil {
return err
}
2022-03-18 02:12:25 +00:00
ok, err := store.client.DatabaseExists(ctx, store.databaseName)
2022-03-17 09:49:26 +00:00
if err != nil {
return err
}
if ok {
2022-03-18 02:12:25 +00:00
store.database, err = store.client.Database(ctx, store.databaseName)
2022-03-17 09:49:26 +00:00
} else {
2022-03-18 02:12:25 +00:00
store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{})
2022-03-17 09:49:26 +00:00
}
if err != nil {
return err
}
if store.kvCollection, err = store.ensureCollection(ctx, KVMETA_COLLECTION); err != nil {
2022-03-18 02:12:25 +00:00
return err
}
2022-03-17 09:49:26 +00:00
return err
}
type key int
const (
transactionKey key = 0
)
func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
keys := make([]string, 0, len(store.buckets)+1)
for k := range store.buckets {
keys = append(keys, k)
}
keys = append(keys, store.kvCollection.Name())
2022-03-17 09:49:26 +00:00
txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{
Exclusive: keys,
2022-03-17 09:49:26 +00:00
}, &driver.BeginTransactionOptions{})
if err != nil {
return nil, err
}
return context.WithValue(ctx, transactionKey, txn), nil
}
func (store *ArangodbStore) CommitTransaction(ctx context.Context) error {
val := ctx.Value(transactionKey)
cast, ok := val.(driver.TransactionID)
if !ok {
return fmt.Errorf("txn cast fail %s:", val)
}
err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{})
if err != nil {
return err
}
return nil
}
func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error {
val := ctx.Value(transactionKey)
cast, ok := val.(driver.TransactionID)
if !ok {
return fmt.Errorf("txn cast fail %s:", val)
}
err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{})
if err != nil {
return err
}
return nil
}
func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
dir, name := entry.FullPath.DirAndName()
meta, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
2022-03-17 09:49:26 +00:00
meta = util.MaybeGzipData(meta)
}
model := &Model{
Key: hashString(string(entry.FullPath)),
Directory: dir,
Name: name,
Meta: bytesToArray(meta),
}
2022-03-18 03:35:30 +00:00
if entry.TtlSec > 0 {
model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339)
} else {
model.Ttl = ""
}
targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath)
if err != nil {
return err
}
_, err = targetCollection.CreateDocument(ctx, model)
2022-03-18 02:12:25 +00:00
if driver.IsConflict(err) {
return store.UpdateEntry(ctx, entry)
}
2022-03-17 09:49:26 +00:00
if err != nil {
2022-03-18 02:12:25 +00:00
return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err)
2022-03-17 09:49:26 +00:00
}
return nil
}
func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
dir, name := entry.FullPath.DirAndName()
meta, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
2022-03-17 09:49:26 +00:00
meta = util.MaybeGzipData(meta)
}
model := &Model{
Key: hashString(string(entry.FullPath)),
Directory: dir,
Name: name,
Meta: bytesToArray(meta),
}
2022-03-18 03:35:30 +00:00
if entry.TtlSec > 0 {
model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339)
} else {
model.Ttl = "none"
}
targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath)
if err != nil {
return err
}
_, err = targetCollection.UpdateDocument(ctx, model.Key, model)
2022-03-17 09:49:26 +00:00
if err != nil {
return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err)
}
return nil
}
func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
var data Model
targetCollection, err := store.extractBucketCollection(ctx, fullpath)
if err != nil {
return nil, err
2022-03-17 09:49:26 +00:00
}
_, err = targetCollection.ReadDocument(ctx, hashString(string(fullpath)), &data)
2022-03-17 09:49:26 +00:00
if err != nil {
if driver.IsNotFound(err) {
return nil, filer_pb.ErrNotFound
}
2022-03-17 09:49:26 +00:00
glog.Errorf("find %s: %v", fullpath, err)
return nil, filer_pb.ErrNotFound
}
if len(data.Meta) == 0 {
return nil, filer_pb.ErrNotFound
}
entry = &filer.Entry{
FullPath: fullpath,
}
err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta)))
if err != nil {
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
}
return entry, nil
}
func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
targetCollection, err := store.extractBucketCollection(ctx, fullpath)
if err != nil {
return err
}
_, err = targetCollection.RemoveDocument(ctx, hashString(string(fullpath)))
2022-03-18 02:12:25 +00:00
if err != nil && !driver.IsNotFound(err) {
2022-03-17 09:49:26 +00:00
glog.Errorf("find %s: %v", fullpath, err)
return fmt.Errorf("delete %s : %v", fullpath, err)
}
return nil
}
// this runs in log time
func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
2022-03-18 05:26:11 +00:00
var query string
targetCollection, err := store.extractBucketCollection(ctx, fullpath)
if err != nil {
return err
}
2022-03-18 05:29:48 +00:00
query = query + fmt.Sprintf(`
for d in %s
2022-03-18 05:29:48 +00:00
filter starts_with(d.directory, "%s/") || d.directory == "%s"
remove d._key in %s`,
targetCollection.Name(),
2022-03-18 02:12:25 +00:00
strings.Join(strings.Split(string(fullpath), "/"), ","),
string(fullpath),
targetCollection.Name(),
2022-03-18 02:12:25 +00:00
)
cur, err := store.database.Query(ctx, query, nil)
2022-03-17 09:49:26 +00:00
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
defer cur.Close()
return nil
}
func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
}
2022-03-17 09:49:26 +00:00
func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
targetCollection, err := store.extractBucketCollection(ctx, dirPath+"/")
if err != nil {
return lastFileName, err
2022-03-18 02:12:25 +00:00
}
query := "for d in " + targetCollection.Name()
2022-03-18 02:12:25 +00:00
if includeStartFile {
query = query + " filter d.name >= \"" + startFileName + "\" "
2022-03-18 02:12:25 +00:00
} else {
query = query + " filter d.name > \"" + startFileName + "\" "
2022-03-18 02:12:25 +00:00
}
if prefix != "" {
query = query + fmt.Sprintf(`&& starts_with(d.name, "%s")`, prefix)
}
query = query + `
2022-03-18 02:12:25 +00:00
filter d.directory == @dir
sort d.name asc
`
if limit > 0 {
query = query + "limit " + strconv.Itoa(int(limit))
2022-03-18 02:12:25 +00:00
}
query = query + "\n return d"
cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath})
2022-03-17 09:49:26 +00:00
if err != nil {
return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
}
defer cur.Close()
for cur.HasMore() {
var data Model
_, err = cur.ReadDocument(ctx, &data)
if err != nil {
break
}
entry := &filer.Entry{
FullPath: util.NewFullPath(data.Directory, data.Name),
2022-03-17 09:49:26 +00:00
}
lastFileName = data.Name
converted := arrayToBytes(data.Meta)
if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
err = decodeErr
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
break
}
if !eachEntryFunc(entry) {
break
}
}
return lastFileName, err
}
func (store *ArangodbStore) Shutdown() {
}