mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
arangodb adapter
This commit is contained in:
elee
parent
68cf6a9229
commit
921535001a
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -75,6 +75,8 @@ com_crashlytics_export_strings.xml
|
|||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
|
||||
workspace/
|
||||
|
||||
test_data
|
||||
build
|
||||
target
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -42,7 +42,7 @@ require (
|
|||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/go-zookeeper/zk v1.0.2 // indirect
|
||||
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
|
|
@ -169,6 +169,8 @@ require (
|
|||
require (
|
||||
cloud.google.com/go/kms v1.0.0 // indirect
|
||||
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
|
||||
github.com/arangodb/go-driver v1.2.1 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
|
|
|
|||
11
go.sum
11
go.sum
|
|
@ -124,6 +124,10 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
|
|||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8=
|
||||
github.com/arangodb/go-driver v1.2.1/go.mod h1:zdDkJJnCj8DAkfbtIjIXnsTrWIiy6VhP3Vy14p+uQeY=
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
|
|
@ -176,6 +180,7 @@ github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636Fk
|
|||
github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
|
|
@ -193,6 +198,7 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4=
|
||||
github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
|
|
@ -290,6 +296,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
|||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
|
|
@ -690,6 +698,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq
|
|||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
|
||||
|
|
@ -1092,6 +1102,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
|
|||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import (
|
|||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
|
||||
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/arangodb"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
|
||||
|
|
|
|||
|
|
@ -285,6 +285,12 @@ healthcheck_enabled = false
|
|||
index.max_result_window = 10000
|
||||
|
||||
|
||||
[arangodb] # in development dont use it
|
||||
enabled = false
|
||||
arango_host=["http://localhost:8529"]
|
||||
arango_user=""
|
||||
arango_pass=""
|
||||
|
||||
|
||||
##########################
|
||||
##########################
|
||||
|
|
|
|||
348
weed/filer/arangodb/arangodb_store.go
Normal file
348
weed/filer/arangodb/arangodb_store.go
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
package arangodb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/arangodb/go-driver"
|
||||
"github.com/arangodb/go-driver/http"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
filer.Stores = append(filer.Stores, &ArangodbStore{})
|
||||
}
|
||||
|
||||
type ArangodbStore struct {
|
||||
connect driver.Connection
|
||||
client driver.Client
|
||||
database driver.Database
|
||||
collection driver.Collection
|
||||
}
|
||||
|
||||
type Model struct {
|
||||
Key string `json:"_key"`
|
||||
Directory string `json:"directory"`
|
||||
Name string `json:"name"`
|
||||
Meta []uint64 `json:"meta"`
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) GetName() string {
|
||||
return "arangodb"
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
|
||||
return store.connection(configuration.GetStringSlice(prefix+"arango_host"),
|
||||
configuration.GetString(prefix+"arango_user"),
|
||||
configuration.GetString(prefix+"arango_pass"),
|
||||
)
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) connection(uris []string, user string, pass string) (err error) {
|
||||
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
|
||||
store.connect, err = http.NewConnection(http.ConnectionConfig{
|
||||
Endpoints: uris,
|
||||
TLSConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
store.client, err = driver.NewClient(driver.ClientConfig{
|
||||
Connection: store.connect,
|
||||
Authentication: driver.BasicAuthentication(user, pass),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db_name := "seaweed-filer"
|
||||
ok, err := store.client.DatabaseExists(ctx, db_name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
store.database, err = store.client.Database(ctx, db_name)
|
||||
} else {
|
||||
store.database, err = store.client.CreateDatabase(ctx, db_name, &driver.CreateDatabaseOptions{})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
coll_name := "files"
|
||||
ok, err = store.database.CollectionExists(ctx, coll_name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
store.collection, err = store.database.Collection(ctx, coll_name)
|
||||
} else {
|
||||
store.collection, err = store.database.CreateCollection(ctx, coll_name, &driver.CreateCollectionOptions{})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// ensure indices
|
||||
|
||||
if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, &driver.EnsurePersistentIndexOptions{
|
||||
Name: "directory_name_multi",
|
||||
Unique: true,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"},
|
||||
&driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory_fulltext"},
|
||||
&driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{
|
||||
Name: "IDX_name",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
type key int
|
||||
|
||||
const (
|
||||
transactionKey key = 0
|
||||
)
|
||||
|
||||
func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
|
||||
txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{
|
||||
Exclusive: []string{"files"},
|
||||
}, &driver.BeginTransactionOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return context.WithValue(ctx, transactionKey, txn), nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) CommitTransaction(ctx context.Context) error {
|
||||
val := ctx.Value(transactionKey)
|
||||
cast, ok := val.(driver.TransactionID)
|
||||
if !ok {
|
||||
return fmt.Errorf("txn cast fail %s:", val)
|
||||
}
|
||||
err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error {
|
||||
val := ctx.Value(transactionKey)
|
||||
cast, ok := val.(driver.TransactionID)
|
||||
if !ok {
|
||||
return fmt.Errorf("txn cast fail %s:", val)
|
||||
}
|
||||
err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
dir, name := entry.FullPath.DirAndName()
|
||||
meta, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
|
||||
}
|
||||
|
||||
if len(entry.Chunks) > 50 {
|
||||
meta = util.MaybeGzipData(meta)
|
||||
}
|
||||
model := &Model{
|
||||
Key: hashString(string(entry.FullPath)),
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
Meta: bytesToArray(meta),
|
||||
}
|
||||
_, err = store.collection.CreateDocument(ctx, model)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
dir, name := entry.FullPath.DirAndName()
|
||||
meta, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
|
||||
}
|
||||
|
||||
if len(entry.Chunks) > 50 {
|
||||
meta = util.MaybeGzipData(meta)
|
||||
}
|
||||
model := &Model{
|
||||
Key: hashString(string(entry.FullPath)),
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
Meta: bytesToArray(meta),
|
||||
}
|
||||
|
||||
_, err = store.collection.UpdateDocument(ctx, model.Key, model)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
|
||||
var data Model
|
||||
_, err = store.collection.ReadDocument(ctx, hashString(string(fullpath)), &data)
|
||||
if driver.IsNotFound(err) {
|
||||
return nil, filer_pb.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorf("find %s: %v", fullpath, err)
|
||||
return nil, filer_pb.ErrNotFound
|
||||
}
|
||||
if len(data.Meta) == 0 {
|
||||
return nil, filer_pb.ErrNotFound
|
||||
}
|
||||
entry = &filer.Entry{
|
||||
FullPath: fullpath,
|
||||
}
|
||||
err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta)))
|
||||
if err != nil {
|
||||
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
|
||||
_, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath)))
|
||||
if err != nil {
|
||||
glog.Errorf("find %s: %v", fullpath, err)
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
|
||||
dir, _ := fullpath.DirAndName()
|
||||
cur, err := store.database.Query(ctx, `
|
||||
for d in files
|
||||
filter d.directory == @dir
|
||||
remove d in files`, map[string]interface{}{"dir": dir})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
defer cur.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
|
||||
return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
|
||||
dir, name := dirPath.DirAndName()
|
||||
eq := ""
|
||||
if includeStartFile {
|
||||
eq = "="
|
||||
}
|
||||
_ = eq
|
||||
_ = name
|
||||
|
||||
cur, err := store.database.Query(ctx, fmt.Sprintf(`
|
||||
for d in files
|
||||
filter d.directory == @dir
|
||||
sort d.name desc
|
||||
limit %d
|
||||
return d`, limit), map[string]interface{}{
|
||||
"dir": dir,
|
||||
})
|
||||
if err != nil {
|
||||
return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
|
||||
}
|
||||
defer cur.Close()
|
||||
for cur.HasMore() {
|
||||
var data Model
|
||||
_, err = cur.ReadDocument(ctx, &data)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
entry := &filer.Entry{
|
||||
FullPath: util.NewFullPath(string(dirPath), data.Name),
|
||||
}
|
||||
lastFileName = data.Name
|
||||
converted := arrayToBytes(data.Meta)
|
||||
if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
|
||||
err = decodeErr
|
||||
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
|
||||
break
|
||||
}
|
||||
|
||||
if !eachEntryFunc(entry) {
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
return lastFileName, err
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) Shutdown() {
|
||||
}
|
||||
|
||||
func hashString(dir string) string {
|
||||
h := md5.New()
|
||||
io.WriteString(h, dir)
|
||||
b := h.Sum(nil)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
func bytesToArray(bs []byte) []uint64 {
|
||||
out := make([]uint64, 0, 2+len(bs)/8)
|
||||
out = append(out, uint64(len(bs)))
|
||||
for len(bs)%8 != 0 {
|
||||
bs = append(bs, 0)
|
||||
}
|
||||
for i := 0; i < len(bs); i = i + 8 {
|
||||
out = append(out, binary.BigEndian.Uint64(bs[i:]))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func arrayToBytes(xs []uint64) []byte {
|
||||
if len(xs) < 2 {
|
||||
return []byte{}
|
||||
}
|
||||
first := xs[0]
|
||||
out := make([]byte, len(xs)*8)
|
||||
for i := 1; i < len(xs); i = i + 1 {
|
||||
binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i])
|
||||
}
|
||||
return out[:first]
|
||||
}
|
||||
62
weed/filer/arangodb/arangodb_store_kv.go
Normal file
62
weed/filer/arangodb/arangodb_store_kv.go
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
package arangodb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
|
||||
dir, name := genDirAndName(key)
|
||||
model := &Model{
|
||||
Key: hashString(string(key)),
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
Meta: bytesToArray(value),
|
||||
}
|
||||
|
||||
exists, err := store.collection.DocumentExists(ctx, model.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("kv put: %v", err)
|
||||
}
|
||||
if exists {
|
||||
_, err = store.collection.UpdateDocument(ctx, model.Key, model)
|
||||
} else {
|
||||
_, err = store.collection.CreateDocument(ctx, model)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("kv put: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
|
||||
var model Model
|
||||
_, err = store.collection.ReadDocument(ctx, hashString(string(key)), &model)
|
||||
if err != nil {
|
||||
glog.Errorf("kv get: %v", err)
|
||||
return nil, filer.ErrKvNotFound
|
||||
}
|
||||
return arrayToBytes(model.Meta), nil
|
||||
}
|
||||
|
||||
func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) {
|
||||
_, err = store.collection.RemoveDocument(ctx, hashString(string(key)))
|
||||
if err != nil {
|
||||
glog.Errorf("kv del: %v", err)
|
||||
return filer.ErrKvNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func genDirAndName(key []byte) (dir string, name string) {
|
||||
for len(key) < 8 {
|
||||
key = append(key, 0)
|
||||
}
|
||||
dir = string(key[:8])
|
||||
name = string(key[8:])
|
||||
return
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/arangodb"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
|
||||
|
|
|
|||
Loading…
Reference in a new issue