From 68cf6a922954587d0457a100a7576afc209f26eb Mon Sep 17 00:00:00 2001 From: aaaaa Date: Thu, 17 Mar 2022 06:07:41 +0000 Subject: [PATCH 01/29] a --- .gitpod.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 000000000..fe8e4d9d9 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,10 @@ +tasks: + - init: go mod tidy + - name: redis + command: docker run -p 6379:6379 redis:6.2.6 + - name: arangodb + command: docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0 + - name: postgresql + command: docker run -p 5432:5432 -e POSTGRES_PASSWORD=test postgres:14.2 + - name: cassandra + command: docker run -p 7000:7000 cassandra:4.0.3 From 921535001a62424791549d51f6c91f3baefd2f1b Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 04:49:26 -0500 Subject: [PATCH 02/29] arangodb adapter --- .gitignore | 2 + go.mod | 4 +- go.sum | 11 + weed/command/imports.go | 1 + weed/command/scaffold/filer.toml | 6 + weed/filer/arangodb/arangodb_store.go | 348 +++++++++++++++++++++++ weed/filer/arangodb/arangodb_store_kv.go | 62 ++++ weed/server/filer_server.go | 1 + 8 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 weed/filer/arangodb/arangodb_store.go create mode 100644 weed/filer/arangodb/arangodb_store_kv.go diff --git a/.gitignore b/.gitignore index 671b01051..e7ae92784 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,8 @@ com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties +workspace/ + test_data build target diff --git a/go.mod b/go.mod index 1507a16ea..862cf7ecb 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/go-stack/stack v1.8.0 // indirect github.com/go-zookeeper/zk v1.0.2 // indirect github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d - github.com/golang-jwt/jwt v3.2.1+incompatible + github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 // indirect @@ -169,6 +169,8 @@ require ( require ( cloud.google.com/go/kms v1.0.0 // indirect github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect + github.com/arangodb/go-driver v1.2.1 // indirect + github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/d4l3k/messagediff v1.2.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 7d3b72eb6..7bec2c973 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,10 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8= +github.com/arangodb/go-driver v1.2.1/go.mod h1:zdDkJJnCj8DAkfbtIjIXnsTrWIiy6VhP3Vy14p+uQeY= +github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= +github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -176,6 +180,7 @@ github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636Fk github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -193,6 +198,7 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -290,6 +296,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -690,6 +698,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -1092,6 +1102,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/weed/command/imports.go b/weed/command/imports.go index 3792c45c4..5b3195907 100644 --- a/weed/command/imports.go +++ b/weed/command/imports.go @@ -15,6 +15,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" + _ "github.com/chrislusf/seaweedfs/weed/filer/arangodb" _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 5d4513c36..19da66dc1 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -285,6 +285,12 @@ healthcheck_enabled = false index.max_result_window = 10000 +[arangodb] # in development dont use it +enabled = false +arango_host=["http://localhost:8529"] +arango_user="" +arango_pass="" + ########################## ########################## diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go new file mode 100644 index 000000000..3cc9e14c9 --- /dev/null +++ b/weed/filer/arangodb/arangodb_store.go @@ -0,0 +1,348 @@ +package arangodb + +import ( + "context" + "crypto/md5" + "crypto/tls" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "time" + + "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func init() { + filer.Stores = append(filer.Stores, &ArangodbStore{}) +} + +type ArangodbStore struct { + connect driver.Connection + client driver.Client + database driver.Database + collection driver.Collection +} + +type Model struct { + Key string `json:"_key"` + Directory string `json:"directory"` + Name string `json:"name"` + Meta []uint64 `json:"meta"` +} + +func (store *ArangodbStore) GetName() string { + return "arangodb" +} + +func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.connection(configuration.GetStringSlice(prefix+"arango_host"), + configuration.GetString(prefix+"arango_user"), + configuration.GetString(prefix+"arango_pass"), + ) +} + +func (store *ArangodbStore) connection(uris []string, user string, pass string) (err error) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + + store.connect, err = http.NewConnection(http.ConnectionConfig{ + Endpoints: uris, + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }) + if err != nil { + return err + } + store.client, err = driver.NewClient(driver.ClientConfig{ + Connection: store.connect, + Authentication: driver.BasicAuthentication(user, pass), + }) + if err != nil { + return err + } + db_name := "seaweed-filer" + ok, err := store.client.DatabaseExists(ctx, db_name) + if err != nil { + return err + } + if ok { + store.database, err = store.client.Database(ctx, db_name) + } else { + store.database, err = store.client.CreateDatabase(ctx, db_name, &driver.CreateDatabaseOptions{}) + } + if err != nil { + return err + } + + coll_name := "files" + ok, err = store.database.CollectionExists(ctx, coll_name) + if err != nil { + return err + } + if ok { + store.collection, err = store.database.Collection(ctx, coll_name) + } else { + store.collection, err = store.database.CreateCollection(ctx, coll_name, &driver.CreateCollectionOptions{}) + } + if err != nil { + return err + } + + // ensure indices + + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, &driver.EnsurePersistentIndexOptions{ + Name: "directory_name_multi", + Unique: true, + }); err != nil { + return err + } + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, + &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { + return err + } + + if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory_fulltext"}, + &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { + return err + } + + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ + Name: "IDX_name", + }); err != nil { + return err + } + + return err +} + +type key int + +const ( + transactionKey key = 0 +) + +func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) { + txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{ + Exclusive: []string{"files"}, + }, &driver.BeginTransactionOptions{}) + if err != nil { + return nil, err + } + + return context.WithValue(ctx, transactionKey, txn), nil +} + +func (store *ArangodbStore) CommitTransaction(ctx context.Context) error { + val := ctx.Value(transactionKey) + cast, ok := val.(driver.TransactionID) + if !ok { + return fmt.Errorf("txn cast fail %s:", val) + } + err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{}) + if err != nil { + return err + } + return nil +} + +func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error { + val := ctx.Value(transactionKey) + cast, ok := val.(driver.TransactionID) + if !ok { + return fmt.Errorf("txn cast fail %s:", val) + } + err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{}) + if err != nil { + return err + } + return nil +} + +func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if len(entry.Chunks) > 50 { + meta = util.MaybeGzipData(meta) + } + model := &Model{ + Key: hashString(string(entry.FullPath)), + Directory: dir, + Name: name, + Meta: bytesToArray(meta), + } + _, err = store.collection.CreateDocument(ctx, model) + + if err != nil { + return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) + } + + return nil + +} + +func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if len(entry.Chunks) > 50 { + meta = util.MaybeGzipData(meta) + } + model := &Model{ + Key: hashString(string(entry.FullPath)), + Directory: dir, + Name: name, + Meta: bytesToArray(meta), + } + + _, err = store.collection.UpdateDocument(ctx, model.Key, model) + + if err != nil { + return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) + } + + return nil +} + +func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + var data Model + _, err = store.collection.ReadDocument(ctx, hashString(string(fullpath)), &data) + if driver.IsNotFound(err) { + return nil, filer_pb.ErrNotFound + } + if err != nil { + glog.Errorf("find %s: %v", fullpath, err) + return nil, filer_pb.ErrNotFound + } + if len(data.Meta) == 0 { + return nil, filer_pb.ErrNotFound + } + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta))) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { + _, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath))) + if err != nil { + glog.Errorf("find %s: %v", fullpath, err) + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + return nil +} + +func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { + dir, _ := fullpath.DirAndName() + cur, err := store.database.Query(ctx, ` +for d in files +filter d.directory == @dir +remove d in files`, map[string]interface{}{"dir": dir}) + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + defer cur.Close() + return nil +} + +func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed +} + +func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + dir, name := dirPath.DirAndName() + eq := "" + if includeStartFile { + eq = "=" + } + _ = eq + _ = name + + cur, err := store.database.Query(ctx, fmt.Sprintf(` +for d in files +filter d.directory == @dir +sort d.name desc +limit %d +return d`, limit), map[string]interface{}{ + "dir": dir, + }) + if err != nil { + return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) + } + defer cur.Close() + for cur.HasMore() { + var data Model + _, err = cur.ReadDocument(ctx, &data) + if err != nil { + break + } + entry := &filer.Entry{ + FullPath: util.NewFullPath(string(dirPath), data.Name), + } + lastFileName = data.Name + converted := arrayToBytes(data.Meta) + if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } + + if !eachEntryFunc(entry) { + break + } + + } + return lastFileName, err +} + +func (store *ArangodbStore) Shutdown() { +} + +func hashString(dir string) string { + h := md5.New() + io.WriteString(h, dir) + b := h.Sum(nil) + return hex.EncodeToString(b) +} + +func bytesToArray(bs []byte) []uint64 { + out := make([]uint64, 0, 2+len(bs)/8) + out = append(out, uint64(len(bs))) + for len(bs)%8 != 0 { + bs = append(bs, 0) + } + for i := 0; i < len(bs); i = i + 8 { + out = append(out, binary.BigEndian.Uint64(bs[i:])) + } + return out +} + +func arrayToBytes(xs []uint64) []byte { + if len(xs) < 2 { + return []byte{} + } + first := xs[0] + out := make([]byte, len(xs)*8) + for i := 1; i < len(xs); i = i + 1 { + binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i]) + } + return out[:first] +} diff --git a/weed/filer/arangodb/arangodb_store_kv.go b/weed/filer/arangodb/arangodb_store_kv.go new file mode 100644 index 000000000..93caa75ed --- /dev/null +++ b/weed/filer/arangodb/arangodb_store_kv.go @@ -0,0 +1,62 @@ +package arangodb + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + dir, name := genDirAndName(key) + model := &Model{ + Key: hashString(string(key)), + Directory: dir, + Name: name, + Meta: bytesToArray(value), + } + + exists, err := store.collection.DocumentExists(ctx, model.Key) + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + if exists { + _, err = store.collection.UpdateDocument(ctx, model.Key, model) + } else { + _, err = store.collection.CreateDocument(ctx, model) + } + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + var model Model + _, err = store.collection.ReadDocument(ctx, hashString(string(key)), &model) + if err != nil { + glog.Errorf("kv get: %v", err) + return nil, filer.ErrKvNotFound + } + return arrayToBytes(model.Meta), nil +} + +func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) { + _, err = store.collection.RemoveDocument(ctx, hashString(string(key))) + if err != nil { + glog.Errorf("kv del: %v", err) + return filer.ErrKvNotFound + } + return nil +} + +func genDirAndName(key []byte) (dir string, name string) { + for len(key) < 8 { + key = append(key, 0) + } + dir = string(key[:8]) + name = string(key[8:]) + return +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 497f59568..e25de4795 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -21,6 +21,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer" + _ "github.com/chrislusf/seaweedfs/weed/filer/arangodb" _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" From 31571dd96e9c3b299dc72d0afac2859098248fc9 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 05:07:40 -0500 Subject: [PATCH 03/29] fix file listing --- weed/filer/arangodb/arangodb_store.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 3cc9e14c9..dc4fbad84 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -268,22 +268,19 @@ func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, di } func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - dir, name := dirPath.DirAndName() eq := "" - if includeStartFile { - eq = "=" + if !includeStartFile { + eq = "filter d.name != \"" + startFileName + "\"" } - _ = eq - _ = name - - cur, err := store.database.Query(ctx, fmt.Sprintf(` + fmt.Println(dirPath, startFileName, includeStartFile) + query := fmt.Sprintf(` for d in files -filter d.directory == @dir +filter d.directory == "%s" sort d.name desc +%s limit %d -return d`, limit), map[string]interface{}{ - "dir": dir, - }) +return d`, string(dirPath), eq, limit) + cur, err := store.database.Query(ctx, query, nil) if err != nil { return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) } From f3ab6769e991554969fde3f29fe170a38d34a996 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 05:09:13 -0500 Subject: [PATCH 04/29] make start from indexing work --- weed/filer/arangodb/arangodb_store.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index dc4fbad84..f10576105 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -263,16 +263,18 @@ remove d in files`, map[string]interface{}{"dir": dir}) return nil } +//TODO: use fulltext index func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { eq := "" - if !includeStartFile { - eq = "filter d.name != \"" + startFileName + "\"" + if includeStartFile { + eq = "filter d.name >= \"" + startFileName + "\"" + } else { + eq = "filter d.name > \"" + startFileName + "\"" } - fmt.Println(dirPath, startFileName, includeStartFile) query := fmt.Sprintf(` for d in files filter d.directory == "%s" From 94883e2fadefae5142bcfa5420e38143d0725ada Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 05:23:12 -0500 Subject: [PATCH 05/29] ok then --- weed/filer/arangodb/arangodb_store.go | 46 +++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index f10576105..7d47ce2d5 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -107,7 +107,7 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) return err } - if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory_fulltext"}, + if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { return err } @@ -263,11 +263,53 @@ remove d in files`, map[string]interface{}{"dir": dir}) return nil } -//TODO: use fulltext index func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } +//func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { +// eq := "" +// if includeStartFile { +// eq = "filter d.name >= \"" + startFileName + "\"" +// } else { +// eq = "filter d.name > \"" + startFileName + "\"" +// } +// query := fmt.Sprintf(` +//for d in fulltext(files,"directory","prefix:%s") +//sort d.name desc +//%s +//limit %d +//return d`, string(dirPath), eq, limit) +// cur, err := store.database.Query(ctx, query, nil) +// if err != nil { +// return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) +// } +// defer cur.Close() +// for cur.HasMore() { +// var data Model +// _, err = cur.ReadDocument(ctx, &data) +// if err != nil { +// break +// } +// entry := &filer.Entry{ +// FullPath: util.NewFullPath(data.Directory, data.Name), +// } +// lastFileName = data.Name +// converted := arrayToBytes(data.Meta) +// if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { +// err = decodeErr +// glog.V(0).Infof("list %s : %v", entry.FullPath, err) +// break +// } +// +// if !eachEntryFunc(entry) { +// break +// } +// +// } +// return lastFileName, err +//} + func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { eq := "" if includeStartFile { From 52ce8cc4f27f8a575147402244e2c084ef58a241 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 14:06:11 -0500 Subject: [PATCH 06/29] remove gitpod yml --- .gitpod.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml deleted file mode 100644 index fe8e4d9d9..000000000 --- a/.gitpod.yml +++ /dev/null @@ -1,10 +0,0 @@ -tasks: - - init: go mod tidy - - name: redis - command: docker run -p 6379:6379 redis:6.2.6 - - name: arangodb - command: docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0 - - name: postgresql - command: docker run -p 5432:5432 -e POSTGRES_PASSWORD=test postgres:14.2 - - name: cassandra - command: docker run -p 7000:7000 cassandra:4.0.3 From a0df993cefd4872430097cbe598ab91b9817aea3 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 14:08:36 -0500 Subject: [PATCH 07/29] cleanup, add a few comments --- weed/filer/arangodb/arangodb_store.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 7d47ce2d5..9095a96f9 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -106,11 +106,12 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } - - if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, - &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { - return err - } + // fulltext index not required since no prefix search + // user should just make one themselves if they intend on using it + // if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, + // &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { + // return err + // } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ Name: "IDX_name", @@ -267,6 +268,8 @@ func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, di return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } +//TODO: i must be misunderstanding what this function is supposed to do +//so figure it out is the todo, i guess lol - aaaaa //func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { // eq := "" // if includeStartFile { @@ -357,6 +360,7 @@ return d`, string(dirPath), eq, limit) func (store *ArangodbStore) Shutdown() { } +//convert a string into arango-key safe hex bytes hash func hashString(dir string) string { h := md5.New() io.WriteString(h, dir) From bf745bdccb3d35c661cf75c10534ece0c0d9d392 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 14:09:34 -0500 Subject: [PATCH 08/29] revise comment --- weed/filer/arangodb/arangodb_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 9095a96f9..e0f036344 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -106,8 +106,8 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } - // fulltext index not required since no prefix search - // user should just make one themselves if they intend on using it + // fulltext index not required since no prefix search + // might change // if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, // &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { // return err From 423ce57cde83433103a4a59c04421c8c33b84287 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 21:12:25 -0500 Subject: [PATCH 09/29] prefix search, bucket implemented --- weed/command/scaffold/filer.toml | 10 +- weed/filer/arangodb/arangodb_store.go | 224 +++++++++---------- weed/filer/arangodb/arangodb_store_bucket.go | 30 +++ weed/filer/arangodb/arangodb_store_kv.go | 26 +-- weed/filer/arangodb/helpers.go | 44 ++++ weed/filer/arangodb/readme.md | 29 +++ weed/filer/filer_delete_entry.go | 2 - 7 files changed, 227 insertions(+), 138 deletions(-) create mode 100644 weed/filer/arangodb/arangodb_store_bucket.go create mode 100644 weed/filer/arangodb/helpers.go create mode 100644 weed/filer/arangodb/readme.md diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 19da66dc1..6a7835de3 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -287,9 +287,13 @@ index.max_result_window = 10000 [arangodb] # in development dont use it enabled = false -arango_host=["http://localhost:8529"] -arango_user="" -arango_pass="" +db_name = "seaweedfs" +servers=["http://localhost:8529"] # list of servers to connect to +# only basic auth supported for now +user="" +pass="" +# skip tls cert validation +insecure_skip_verify = true ########################## diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index e0f036344..a9f55f5bc 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -2,12 +2,9 @@ package arangodb import ( "context" - "crypto/md5" "crypto/tls" - "encoding/binary" - "encoding/hex" "fmt" - "io" + "strings" "time" "github.com/arangodb/go-driver" @@ -27,13 +24,20 @@ type ArangodbStore struct { client driver.Client database driver.Database collection driver.Collection + + databaseName string } type Model struct { - Key string `json:"_key"` - Directory string `json:"directory"` - Name string `json:"name"` - Meta []uint64 `json:"meta"` + Key string `json:"_key"` + Directory string `json:"directory,omitempty"` + Name string `json:"name,omitempty"` + Bucket string `json:"bucket,omitempty"` + + //arangodb does not support binary blobs + //we encode byte slice into uint64 slice + //see helpers.go + Meta []uint64 `json:"meta"` } func (store *ArangodbStore) GetName() string { @@ -41,19 +45,21 @@ func (store *ArangodbStore) GetName() string { } func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) { - return store.connection(configuration.GetStringSlice(prefix+"arango_host"), - configuration.GetString(prefix+"arango_user"), - configuration.GetString(prefix+"arango_pass"), + store.databaseName = configuration.GetString(prefix + "db_name") + return store.connection(configuration.GetStringSlice(prefix+"servers"), + configuration.GetString(prefix+"user"), + configuration.GetString(prefix+"pass"), + configuration.GetBool(prefix+"insecure_skip_verify"), ) } -func (store *ArangodbStore) connection(uris []string, user string, pass string) (err error) { +func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) { ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) store.connect, err = http.NewConnection(http.ConnectionConfig{ Endpoints: uris, TLSConfig: &tls.Config{ - InsecureSkipVerify: true, + InsecureSkipVerify: insecure, }, }) if err != nil { @@ -66,15 +72,14 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) if err != nil { return err } - db_name := "seaweed-filer" - ok, err := store.client.DatabaseExists(ctx, db_name) + ok, err := store.client.DatabaseExists(ctx, store.databaseName) if err != nil { return err } if ok { - store.database, err = store.client.Database(ctx, db_name) + store.database, err = store.client.Database(ctx, store.databaseName) } else { - store.database, err = store.client.CreateDatabase(ctx, db_name, &driver.CreateDatabaseOptions{}) + store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{}) } if err != nil { return err @@ -96,29 +101,29 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) // ensure indices - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, &driver.EnsurePersistentIndexOptions{ - Name: "directory_name_multi", - Unique: true, - }); err != nil { + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, + &driver.EnsurePersistentIndexOptions{ + Name: "directory_name_multi", + Unique: true, + }); err != nil { return err } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } - // fulltext index not required since no prefix search - // might change - // if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, - // &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { - // return err - // } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ Name: "IDX_name", }); err != nil { return err } - + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"bucket"}, &driver.EnsurePersistentIndexOptions{ + Name: "IDX_bucket", + Sparse: true, //sparse index, to locate files of bucket + }); err != nil { + return err + } return err } @@ -175,22 +180,41 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) if len(entry.Chunks) > 50 { meta = util.MaybeGzipData(meta) } + bucket, _ := extractBucket(entry.FullPath) model := &Model{ Key: hashString(string(entry.FullPath)), Directory: dir, Name: name, Meta: bytesToArray(meta), + Bucket: bucket, } _, err = store.collection.CreateDocument(ctx, model) - + if driver.IsConflict(err) { + return store.UpdateEntry(ctx, entry) + } if err != nil { - return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) + return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err) } return nil } +func extractBucket(fullpath util.FullPath) (string, string) { + if !strings.HasPrefix(string(fullpath), "/buckets/") { + return "", string(fullpath) + } + bucketAndObjectKey := string(fullpath)[len("/buckets/"):] + t := strings.Index(bucketAndObjectKey, "/") + bucket := bucketAndObjectKey + shortPath := "/" + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = string(util.FullPath(bucketAndObjectKey[t:])) + } + return bucket, shortPath +} + func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -243,20 +267,20 @@ func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPat func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { _, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath))) - if err != nil { + if err != nil && !driver.IsNotFound(err) { glog.Errorf("find %s: %v", fullpath, err) return fmt.Errorf("delete %s : %v", fullpath, err) } - return nil } func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { - dir, _ := fullpath.DirAndName() - cur, err := store.database.Query(ctx, ` -for d in files -filter d.directory == @dir -remove d in files`, map[string]interface{}{"dir": dir}) + query := "" + query = fmt.Sprintf(`for d in files filter starts_with(d.directory, "%s") remove d._key in files`, + strings.Join(strings.Split(string(fullpath), "/"), ","), + string(fullpath), + ) + cur, err := store.database.Query(ctx, query, nil) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -265,53 +289,53 @@ remove d in files`, map[string]interface{}{"dir": dir}) } func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed -} + // if no prefix, then dont use index + if prefix == "" { + return store.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) + } + eq := "" + if includeStartFile { + eq = "filter d.name >= \"" + startFileName + "\"" + } else { + eq = "filter d.name > \"" + startFileName + "\"" + } + query := fmt.Sprintf(` +for d in files +filter d.directory == @dir +filter starts_with(d.name, @prefix) +%s +sort d.name asc +limit %d +return d`, eq, limit) + cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath, "prefix": prefix}) + if err != nil { + return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) + } + defer cur.Close() + for cur.HasMore() { + var data Model + _, err = cur.ReadDocument(ctx, &data) + if err != nil { + break + } + entry := &filer.Entry{ + FullPath: util.NewFullPath(data.Directory, data.Name), + } + lastFileName = data.Name + converted := arrayToBytes(data.Meta) + if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } -//TODO: i must be misunderstanding what this function is supposed to do -//so figure it out is the todo, i guess lol - aaaaa -//func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { -// eq := "" -// if includeStartFile { -// eq = "filter d.name >= \"" + startFileName + "\"" -// } else { -// eq = "filter d.name > \"" + startFileName + "\"" -// } -// query := fmt.Sprintf(` -//for d in fulltext(files,"directory","prefix:%s") -//sort d.name desc -//%s -//limit %d -//return d`, string(dirPath), eq, limit) -// cur, err := store.database.Query(ctx, query, nil) -// if err != nil { -// return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) -// } -// defer cur.Close() -// for cur.HasMore() { -// var data Model -// _, err = cur.ReadDocument(ctx, &data) -// if err != nil { -// break -// } -// entry := &filer.Entry{ -// FullPath: util.NewFullPath(data.Directory, data.Name), -// } -// lastFileName = data.Name -// converted := arrayToBytes(data.Meta) -// if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { -// err = decodeErr -// glog.V(0).Infof("list %s : %v", entry.FullPath, err) -// break -// } -// -// if !eachEntryFunc(entry) { -// break -// } -// -// } -// return lastFileName, err -//} + if !eachEntryFunc(entry) { + break + } + + } + return lastFileName, err +} func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { eq := "" @@ -323,8 +347,8 @@ func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath ut query := fmt.Sprintf(` for d in files filter d.directory == "%s" -sort d.name desc %s +sort d.name asc limit %d return d`, string(dirPath), eq, limit) cur, err := store.database.Query(ctx, query, nil) @@ -359,35 +383,3 @@ return d`, string(dirPath), eq, limit) func (store *ArangodbStore) Shutdown() { } - -//convert a string into arango-key safe hex bytes hash -func hashString(dir string) string { - h := md5.New() - io.WriteString(h, dir) - b := h.Sum(nil) - return hex.EncodeToString(b) -} - -func bytesToArray(bs []byte) []uint64 { - out := make([]uint64, 0, 2+len(bs)/8) - out = append(out, uint64(len(bs))) - for len(bs)%8 != 0 { - bs = append(bs, 0) - } - for i := 0; i < len(bs); i = i + 8 { - out = append(out, binary.BigEndian.Uint64(bs[i:])) - } - return out -} - -func arrayToBytes(xs []uint64) []byte { - if len(xs) < 2 { - return []byte{} - } - first := xs[0] - out := make([]byte, len(xs)*8) - for i := 1; i < len(xs); i = i + 1 { - binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i]) - } - return out[:first] -} diff --git a/weed/filer/arangodb/arangodb_store_bucket.go b/weed/filer/arangodb/arangodb_store_bucket.go new file mode 100644 index 000000000..907328bda --- /dev/null +++ b/weed/filer/arangodb/arangodb_store_bucket.go @@ -0,0 +1,30 @@ +package arangodb + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "time" + + "github.com/chrislusf/seaweedfs/weed/glog" +) + +var _ filer.BucketAware = (*ArangodbStore)(nil) + +func (store *ArangodbStore) OnBucketCreation(bucket string) { + //nothing needs to be done +} +func (store *ArangodbStore) OnBucketDeletion(bucket string) { + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cur, err := store.database.Query(timeout, ` +for d in files +filter d.bucket == @bucket +remove d in files`, map[string]interface{}{"bucket": bucket}) + if err != nil { + glog.V(0).Infof("bucket delete %s : %v", bucket, err) + } + defer cur.Close() +} +func (store *ArangodbStore) CanDropWholeBucket() bool { + return true +} diff --git a/weed/filer/arangodb/arangodb_store_kv.go b/weed/filer/arangodb/arangodb_store_kv.go index 93caa75ed..2978f3dbe 100644 --- a/weed/filer/arangodb/arangodb_store_kv.go +++ b/weed/filer/arangodb/arangodb_store_kv.go @@ -4,16 +4,15 @@ import ( "context" "fmt" + "github.com/arangodb/go-driver" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" ) func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - dir, name := genDirAndName(key) model := &Model{ - Key: hashString(string(key)), - Directory: dir, - Name: name, + Key: hashString(".kvstore." + string(key)), + Directory: ".kvstore." + string(key), Meta: bytesToArray(value), } @@ -32,31 +31,24 @@ func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) return nil } - func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { var model Model - _, err = store.collection.ReadDocument(ctx, hashString(string(key)), &model) + _, err = store.collection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model) + if driver.IsNotFound(err) { + return nil, filer.ErrKvNotFound + } if err != nil { - glog.Errorf("kv get: %v", err) + glog.Errorf("kv get: %s %v", string(key), err) return nil, filer.ErrKvNotFound } return arrayToBytes(model.Meta), nil } func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.collection.RemoveDocument(ctx, hashString(string(key))) + _, err = store.collection.RemoveDocument(ctx, hashString(".kvstore."+string(key))) if err != nil { glog.Errorf("kv del: %v", err) return filer.ErrKvNotFound } return nil } - -func genDirAndName(key []byte) (dir string, name string) { - for len(key) < 8 { - key = append(key, 0) - } - dir = string(key[:8]) - name = string(key[8:]) - return -} diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go new file mode 100644 index 000000000..cf59957a6 --- /dev/null +++ b/weed/filer/arangodb/helpers.go @@ -0,0 +1,44 @@ +package arangodb + +import ( + "crypto/md5" + "encoding/binary" + "encoding/hex" + "io" +) + +//convert a string into arango-key safe hex bytes hash +func hashString(dir string) string { + h := md5.New() + io.WriteString(h, dir) + b := h.Sum(nil) + return hex.EncodeToString(b) +} + +// convert slice of bytes into slice of uint64 +// the first uint64 indicates the length in bytes +func bytesToArray(bs []byte) []uint64 { + out := make([]uint64, 0, 2+len(bs)/8) + out = append(out, uint64(len(bs))) + for len(bs)%8 != 0 { + bs = append(bs, 0) + } + for i := 0; i < len(bs); i = i + 8 { + out = append(out, binary.BigEndian.Uint64(bs[i:])) + } + return out +} + +// convert from slice of uint64 back to bytes +// if input length is 0 or 1, will return nil +func arrayToBytes(xs []uint64) []byte { + if len(xs) < 2 { + return nil + } + first := xs[0] + out := make([]byte, len(xs)*8) // i think this can actually be len(xs)*8-8, but i dont think an extra 8 bytes hurts... + for i := 1; i < len(xs); i = i + 1 { + binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i]) + } + return out[:first] +} diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md new file mode 100644 index 000000000..6cd187305 --- /dev/null +++ b/weed/filer/arangodb/readme.md @@ -0,0 +1,29 @@ +##arangodb + +database: https://github.com/arangodb/arangodb +go driver: https://github.com/arangodb/go-driver + + +options: + +``` +[arangodb] +enabled=true +db_name="seaweedfs" +servers=["http://localhost:8529"] +user="root" +pass="test" + +# whether to enable fulltext index +# this allows for directory prefix query +fulltext=true + +# tls settings +insecure_skip_verify=true +``` + +supports buckets with an extra field in document. +omitempty means extra space is not used. + +i test with +`docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0` diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index c774f5d27..27e68433d 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -25,9 +25,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR if findErr != nil { return findErr } - isDeleteCollection := f.isBucket(entry) - if entry.IsDirectory() { // delete the folder children, not including the folder itself err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(chunks []*filer_pb.FileChunk) error { From 2f0cdcdceb945be7490ec41ba857abcba133e892 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 21:26:16 -0500 Subject: [PATCH 10/29] update readme --- weed/filer/arangodb/readme.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index 6cd187305..24f3a1e7e 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -27,3 +27,22 @@ omitempty means extra space is not used. i test with `docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0` + + +## todo + +implement native TTL + + +## thoughts + +should there be one collection per bucket? would make deleting a bucket instant as compared to fast + + +## comparison + +arangodb uses rocksdb in the background, so i am assuming things run in log time + +single document retreval might run in constant time + +i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion From 171c27ddf1a3889304ae36892a53c43076b5a4cb Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 22:35:30 -0500 Subject: [PATCH 11/29] zz --- weed/filer/arangodb/arangodb_store.go | 16 ++++++++++++++++ weed/filer/arangodb/readme.md | 3 +-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index a9f55f5bc..a39735108 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -33,6 +33,7 @@ type Model struct { Directory string `json:"directory,omitempty"` Name string `json:"name,omitempty"` Bucket string `json:"bucket,omitempty"` + Ttl string `json:"ttl,omitempty"` //arangodb does not support binary blobs //we encode byte slice into uint64 slice @@ -108,10 +109,15 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string, }); err != nil { return err } + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } + if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1, + &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { + return err + } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ Name: "IDX_name", @@ -188,6 +194,11 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) Meta: bytesToArray(meta), Bucket: bucket, } + if entry.TtlSec > 0 { + model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339) + } else { + model.Ttl = "" + } _, err = store.collection.CreateDocument(ctx, model) if driver.IsConflict(err) { return store.UpdateEntry(ctx, entry) @@ -231,6 +242,11 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) Name: name, Meta: bytesToArray(meta), } + if entry.TtlSec > 0 { + model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339) + } else { + model.Ttl = "none" + } _, err = store.collection.UpdateDocument(ctx, model.Key, model) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index 24f3a1e7e..eb6411c2e 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -43,6 +43,5 @@ should there be one collection per bucket? would make deleting a bucket instant arangodb uses rocksdb in the background, so i am assuming things run in log time -single document retreval might run in constant time - i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion +might need to change that From 0701feeb1791abefaad47e585170950922cf792d Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:26:11 -0500 Subject: [PATCH 12/29] added more to readme --- weed/filer/arangodb/arangodb_store.go | 6 +++--- weed/filer/arangodb/readme.md | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index a39735108..c1445f026 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -109,11 +109,11 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string, }); err != nil { return err } - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } + if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1, &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { return err @@ -291,8 +291,8 @@ func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullP } func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { - query := "" - query = fmt.Sprintf(`for d in files filter starts_with(d.directory, "%s") remove d._key in files`, + var query string + query = query + fmt.Sprintf(`filter starts_with(d.directory, "%s") remove d._key in files`, strings.Join(strings.Split(string(fullpath), "/"), ","), string(fullpath), ) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index eb6411c2e..eaa8bd7a7 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -44,4 +44,9 @@ should there be one collection per bucket? would make deleting a bucket instant arangodb uses rocksdb in the background, so i am assuming things run in log time i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion -might need to change that +might need to change that. + + +ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct +it should be log time to the number of files in the directory +and constant time if you have full directory + file From 06f23aa6759155a6d5b713d8704a90e2e2d246d8 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:29:48 -0500 Subject: [PATCH 13/29] put in delete folder children query --- weed/filer/arangodb/arangodb_store.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index c1445f026..23aefe40f 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -292,7 +292,10 @@ func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullP func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { var query string - query = query + fmt.Sprintf(`filter starts_with(d.directory, "%s") remove d._key in files`, + query = query + fmt.Sprintf(` + for d in files + filter starts_with(d.directory, "%s/") || d.directory == "%s" + remove d._key in files`, strings.Join(strings.Split(string(fullpath), "/"), ","), string(fullpath), ) From 25be96832a910394a89e3c70f87edc4a34379a3e Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:33:08 -0500 Subject: [PATCH 14/29] update readme with index info --- weed/filer/arangodb/readme.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index eaa8bd7a7..966fec041 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -41,12 +41,10 @@ should there be one collection per bucket? would make deleting a bucket instant ## comparison -arangodb uses rocksdb in the background, so i am assuming things run in log time - -i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion -might need to change that. - - ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct + it should be log time to the number of files in the directory + and constant time if you have full directory + file + +deleting a folder should be log time to number of folders + files that need to be deleted From 1cea6c73d30ddee95b42335af74701a61c19ec84 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:34:19 -0500 Subject: [PATCH 15/29] update readme --- weed/filer/arangodb/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index 966fec041..e56012d8c 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -31,12 +31,12 @@ i test with ## todo -implement native TTL +performance test ## thoughts -should there be one collection per bucket? would make deleting a bucket instant as compared to fast +should there be one collection per bucket? this would make deleting a bucket O(1) instead of O(n) ## comparison From 411c0df3fec3344c3e5538cf56b54c1567162386 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 21:51:16 -0500 Subject: [PATCH 16/29] switch to multi collection, change readme --- weed/filer/arangodb/arangodb_store.go | 215 +++++++------------ weed/filer/arangodb/arangodb_store_bucket.go | 26 ++- weed/filer/arangodb/arangodb_store_kv.go | 10 +- weed/filer/arangodb/helpers.go | 92 ++++++++ weed/filer/arangodb/readme.md | 44 ++-- 5 files changed, 217 insertions(+), 170 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 23aefe40f..d27799b0e 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -4,7 +4,9 @@ import ( "context" "crypto/tls" "fmt" + "strconv" "strings" + "sync" "time" "github.com/arangodb/go-driver" @@ -19,11 +21,20 @@ func init() { filer.Stores = append(filer.Stores, &ArangodbStore{}) } +var ( + BUCKET_PREFIX = "/buckets" + DEFAULT_COLLECTION = "seaweed_no_bucket" + KVMETA_COLLECTION = "seaweed_kvmeta" +) + type ArangodbStore struct { - connect driver.Connection - client driver.Client - database driver.Database - collection driver.Collection + connect driver.Connection + client driver.Client + database driver.Database + kvCollection driver.Collection + + buckets map[string]driver.Collection + mu sync.RWMutex databaseName string } @@ -32,7 +43,6 @@ type Model struct { Key string `json:"_key"` Directory string `json:"directory,omitempty"` Name string `json:"name,omitempty"` - Bucket string `json:"bucket,omitempty"` Ttl string `json:"ttl,omitempty"` //arangodb does not support binary blobs @@ -46,6 +56,7 @@ func (store *ArangodbStore) GetName() string { } func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) { + store.buckets = make(map[string]driver.Collection, 3) store.databaseName = configuration.GetString(prefix + "db_name") return store.connection(configuration.GetStringSlice(prefix+"servers"), configuration.GetString(prefix+"user"), @@ -85,49 +96,7 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string, if err != nil { return err } - - coll_name := "files" - ok, err = store.database.CollectionExists(ctx, coll_name) - if err != nil { - return err - } - if ok { - store.collection, err = store.database.Collection(ctx, coll_name) - } else { - store.collection, err = store.database.CreateCollection(ctx, coll_name, &driver.CreateCollectionOptions{}) - } - if err != nil { - return err - } - - // ensure indices - - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, - &driver.EnsurePersistentIndexOptions{ - Name: "directory_name_multi", - Unique: true, - }); err != nil { - return err - } - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, - &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { - return err - } - - if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1, - &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { - return err - } - - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ - Name: "IDX_name", - }); err != nil { - return err - } - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"bucket"}, &driver.EnsurePersistentIndexOptions{ - Name: "IDX_bucket", - Sparse: true, //sparse index, to locate files of bucket - }); err != nil { + if store.kvCollection, err = store.ensureCollection(ctx, KVMETA_COLLECTION); err != nil { return err } return err @@ -140,8 +109,13 @@ const ( ) func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) { + keys := make([]string, 0, len(store.buckets)+1) + for k := range store.buckets { + keys = append(keys, k) + } + keys = append(keys, store.kvCollection.Name()) txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{ - Exclusive: []string{"files"}, + Exclusive: keys, }, &driver.BeginTransactionOptions{}) if err != nil { return nil, err @@ -186,23 +160,27 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) if len(entry.Chunks) > 50 { meta = util.MaybeGzipData(meta) } - bucket, _ := extractBucket(entry.FullPath) model := &Model{ Key: hashString(string(entry.FullPath)), Directory: dir, Name: name, Meta: bytesToArray(meta), - Bucket: bucket, } if entry.TtlSec > 0 { model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339) } else { model.Ttl = "" } - _, err = store.collection.CreateDocument(ctx, model) + + targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath) + if err != nil { + return err + } + _, err = targetCollection.CreateDocument(ctx, model) if driver.IsConflict(err) { return store.UpdateEntry(ctx, entry) } + if err != nil { return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err) } @@ -211,21 +189,6 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) } -func extractBucket(fullpath util.FullPath) (string, string) { - if !strings.HasPrefix(string(fullpath), "/buckets/") { - return "", string(fullpath) - } - bucketAndObjectKey := string(fullpath)[len("/buckets/"):] - t := strings.Index(bucketAndObjectKey, "/") - bucket := bucketAndObjectKey - shortPath := "/" - if t > 0 { - bucket = bucketAndObjectKey[:t] - shortPath = string(util.FullPath(bucketAndObjectKey[t:])) - } - return bucket, shortPath -} - func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -247,9 +210,11 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) } else { model.Ttl = "none" } - - _, err = store.collection.UpdateDocument(ctx, model.Key, model) - + targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath) + if err != nil { + return err + } + _, err = targetCollection.UpdateDocument(ctx, model.Key, model) if err != nil { return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) } @@ -259,11 +224,15 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { var data Model - _, err = store.collection.ReadDocument(ctx, hashString(string(fullpath)), &data) - if driver.IsNotFound(err) { - return nil, filer_pb.ErrNotFound - } + targetCollection, err := store.extractBucketCollection(ctx, fullpath) if err != nil { + return nil, err + } + _, err = targetCollection.ReadDocument(ctx, hashString(string(fullpath)), &data) + if err != nil { + if driver.IsNotFound(err) { + return nil, filer_pb.ErrNotFound + } glog.Errorf("find %s: %v", fullpath, err) return nil, filer_pb.ErrNotFound } @@ -281,8 +250,12 @@ func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPat return entry, nil } -func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { - _, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath))) +func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + targetCollection, err := store.extractBucketCollection(ctx, fullpath) + if err != nil { + return err + } + _, err = targetCollection.RemoveDocument(ctx, hashString(string(fullpath))) if err != nil && !driver.IsNotFound(err) { glog.Errorf("find %s: %v", fullpath, err) return fmt.Errorf("delete %s : %v", fullpath, err) @@ -290,14 +263,21 @@ func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullP return nil } -func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { +// this runs in log time +func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { var query string + targetCollection, err := store.extractBucketCollection(ctx, fullpath) + if err != nil { + return err + } query = query + fmt.Sprintf(` - for d in files + for d in %s filter starts_with(d.directory, "%s/") || d.directory == "%s" - remove d._key in files`, + remove d._key in %s`, + targetCollection.Name(), strings.Join(strings.Split(string(fullpath), "/"), ","), string(fullpath), + targetCollection.Name(), ) cur, err := store.database.Query(ctx, query, nil) if err != nil { @@ -307,26 +287,33 @@ func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath u return nil } +func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) +} + func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - // if no prefix, then dont use index - if prefix == "" { - return store.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) + targetCollection, err := store.extractBucketCollection(ctx, dirPath) + if err != nil { + return lastFileName, err } - eq := "" + query := "for d in " + targetCollection.Name() if includeStartFile { - eq = "filter d.name >= \"" + startFileName + "\"" + query = query + " filter d.name >= \"" + startFileName + "\" " } else { - eq = "filter d.name > \"" + startFileName + "\"" + query = query + " filter d.name > \"" + startFileName + "\" " } - query := fmt.Sprintf(` -for d in files + if prefix != "" { + query = query + fmt.Sprintf(`&& starts_with(d.name, "%s")`, prefix) + } + query = query + ` filter d.directory == @dir -filter starts_with(d.name, @prefix) -%s sort d.name asc -limit %d -return d`, eq, limit) - cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath, "prefix": prefix}) +` + if limit > 0 { + query = query + "limit " + strconv.Itoa(int(limit)) + } + query = query + "\n return d" + cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath}) if err != nil { return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) } @@ -356,49 +343,5 @@ return d`, eq, limit) return lastFileName, err } -func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - eq := "" - if includeStartFile { - eq = "filter d.name >= \"" + startFileName + "\"" - } else { - eq = "filter d.name > \"" + startFileName + "\"" - } - query := fmt.Sprintf(` -for d in files -filter d.directory == "%s" -%s -sort d.name asc -limit %d -return d`, string(dirPath), eq, limit) - cur, err := store.database.Query(ctx, query, nil) - if err != nil { - return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) - } - defer cur.Close() - for cur.HasMore() { - var data Model - _, err = cur.ReadDocument(ctx, &data) - if err != nil { - break - } - entry := &filer.Entry{ - FullPath: util.NewFullPath(string(dirPath), data.Name), - } - lastFileName = data.Name - converted := arrayToBytes(data.Meta) - if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { - err = decodeErr - glog.V(0).Infof("list %s : %v", entry.FullPath, err) - break - } - - if !eachEntryFunc(entry) { - break - } - - } - return lastFileName, err -} - func (store *ArangodbStore) Shutdown() { } diff --git a/weed/filer/arangodb/arangodb_store_bucket.go b/weed/filer/arangodb/arangodb_store_bucket.go index 907328bda..63d407309 100644 --- a/weed/filer/arangodb/arangodb_store_bucket.go +++ b/weed/filer/arangodb/arangodb_store_bucket.go @@ -2,28 +2,38 @@ package arangodb import ( "context" - "github.com/chrislusf/seaweedfs/weed/filer" "time" + "github.com/arangodb/go-driver" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" ) var _ filer.BucketAware = (*ArangodbStore)(nil) func (store *ArangodbStore) OnBucketCreation(bucket string) { - //nothing needs to be done + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // create the collection && add to cache + _, err := store.ensureBucket(timeout, bucket) + if err != nil { + glog.V(0).Infof("bucket create %s : %w", bucket, err) + } } func (store *ArangodbStore) OnBucketDeletion(bucket string) { timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - cur, err := store.database.Query(timeout, ` -for d in files -filter d.bucket == @bucket -remove d in files`, map[string]interface{}{"bucket": bucket}) + collection, err := store.ensureBucket(timeout, bucket) if err != nil { - glog.V(0).Infof("bucket delete %s : %v", bucket, err) + glog.V(0).Infof("bucket delete %s : %w", bucket, err) + return + } + err = collection.Remove(timeout) + if err != nil && !driver.IsNotFound(err) { + glog.V(0).Infof("bucket delete %s : %w", bucket, err) + return } - defer cur.Close() } func (store *ArangodbStore) CanDropWholeBucket() bool { return true diff --git a/weed/filer/arangodb/arangodb_store_kv.go b/weed/filer/arangodb/arangodb_store_kv.go index 2978f3dbe..c1307e78d 100644 --- a/weed/filer/arangodb/arangodb_store_kv.go +++ b/weed/filer/arangodb/arangodb_store_kv.go @@ -16,14 +16,14 @@ func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) Meta: bytesToArray(value), } - exists, err := store.collection.DocumentExists(ctx, model.Key) + exists, err := store.kvCollection.DocumentExists(ctx, model.Key) if err != nil { return fmt.Errorf("kv put: %v", err) } if exists { - _, err = store.collection.UpdateDocument(ctx, model.Key, model) + _, err = store.kvCollection.UpdateDocument(ctx, model.Key, model) } else { - _, err = store.collection.CreateDocument(ctx, model) + _, err = store.kvCollection.CreateDocument(ctx, model) } if err != nil { return fmt.Errorf("kv put: %v", err) @@ -33,7 +33,7 @@ func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) } func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { var model Model - _, err = store.collection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model) + _, err = store.kvCollection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model) if driver.IsNotFound(err) { return nil, filer.ErrKvNotFound } @@ -45,7 +45,7 @@ func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte } func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.collection.RemoveDocument(ctx, hashString(".kvstore."+string(key))) + _, err = store.kvCollection.RemoveDocument(ctx, hashString(".kvstore."+string(key))) if err != nil { glog.Errorf("kv del: %v", err) return filer.ErrKvNotFound diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go index cf59957a6..c91ef2be5 100644 --- a/weed/filer/arangodb/helpers.go +++ b/weed/filer/arangodb/helpers.go @@ -1,10 +1,15 @@ package arangodb import ( + "context" "crypto/md5" "encoding/binary" "encoding/hex" "io" + "strings" + + "github.com/arangodb/go-driver" + "github.com/chrislusf/seaweedfs/weed/util" ) //convert a string into arango-key safe hex bytes hash @@ -42,3 +47,90 @@ func arrayToBytes(xs []uint64) []byte { } return out[:first] } + +// gets the bucket name out of filepath +func extractBucket(fullpath util.FullPath) (string, string) { + if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") { + return "", string(fullpath) + } + if strings.Count(string(fullpath), "/") < 2 { + return "", string(fullpath) + } + bucketAndObjectKey := string(fullpath)[len("/buckets/"):] + t := strings.Index(bucketAndObjectKey, "/") + bucket := bucketAndObjectKey + shortPath := "/" + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = string(util.FullPath(bucketAndObjectKey[t:])) + } + return bucket, shortPath +} + +// gets the collection the bucket points to from filepath +func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) { + bucket, _ := extractBucket(fullpath) + if bucket == "" { + bucket = DEFAULT_COLLECTION + } + c, err = store.ensureBucket(ctx, bucket) + if err != nil { + return nil, err + } + return c, err +} + +// get bucket collection from cache. if not exist, creates the buckets collection and grab it +func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) { + var ok bool + store.mu.RLock() + bc, ok = store.buckets[bucket] + store.mu.RUnlock() + if ok { + return bc, nil + } + store.mu.Lock() + defer store.mu.Unlock() + store.buckets[bucket], err = store.ensureCollection(ctx, bucket) + if err != nil { + return nil, err + } + return store.buckets[bucket], nil +} + +// creates collection if not exist, ensures indices if not exist +func (store *ArangodbStore) ensureCollection(ctx context.Context, name string) (c driver.Collection, err error) { + ok, err := store.database.CollectionExists(ctx, name) + if err != nil { + return + } + if ok { + c, err = store.database.Collection(ctx, name) + } else { + c, err = store.database.CreateCollection(ctx, name, &driver.CreateCollectionOptions{}) + } + if err != nil { + return + } + // ensure indices + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory", "name"}, + &driver.EnsurePersistentIndexOptions{ + Name: "directory_name_multi", Unique: true, + }); err != nil { + return + } + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory"}, + &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { + return + } + if _, _, err = c.EnsureTTLIndex(ctx, "ttl", 1, + &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { + return + } + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ + Name: "IDX_name", + }); err != nil { + return + } + return c, nil +} diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index e56012d8c..e189811fb 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -3,7 +3,6 @@ database: https://github.com/arangodb/arangodb go driver: https://github.com/arangodb/go-driver - options: ``` @@ -11,40 +10,43 @@ options: enabled=true db_name="seaweedfs" servers=["http://localhost:8529"] +#basic auth user="root" pass="test" -# whether to enable fulltext index -# this allows for directory prefix query -fulltext=true - # tls settings insecure_skip_verify=true ``` -supports buckets with an extra field in document. -omitempty means extra space is not used. - -i test with +i test using this dev database: `docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0` -## todo - -performance test +## features i don't personally need but are missing + [ ] provide tls cert to arango + [ ] authentication that is not basic auth + [ ] synchronise endpoint interval config + [ ] automatic creation of custom index + [ ] configure default arangodb collection sharding rules + [ ] configure default arangodb collection replication rules -## thoughts - -should there be one collection per bucket? this would make deleting a bucket O(1) instead of O(n) - - -## comparison +## complexity ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct -it should be log time to the number of files in the directory +O(1) +- InsertEntry +- UpdateEntry +- FindEntry +- DeleteEntry +- KvPut +- KvGet +- KvDelete -and constant time if you have full directory + file +O(log(BUCKET_SIZE)) +- DeleteFolderChildren -deleting a folder should be log time to number of folders + files that need to be deleted +O(log(DIRECTORY_SIZE)) +- ListDirectoryEntries +- ListDirectoryPrefixedEntries From beb406bbbb5e67fc66a02d264c670a8fe11785b7 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 22:21:57 -0500 Subject: [PATCH 17/29] fix ls, onBucketDelete still not triggering --- weed/filer/arangodb/arangodb_store.go | 2 +- weed/filer/arangodb/helpers.go | 38 +++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index d27799b0e..27ed9132b 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -292,7 +292,7 @@ func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath ut } func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - targetCollection, err := store.extractBucketCollection(ctx, dirPath) + targetCollection, err := store.extractBucketCollection(ctx, dirPath+"/") if err != nil { return lastFileName, err } diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go index c91ef2be5..943189781 100644 --- a/weed/filer/arangodb/helpers.go +++ b/weed/filer/arangodb/helpers.go @@ -48,25 +48,6 @@ func arrayToBytes(xs []uint64) []byte { return out[:first] } -// gets the bucket name out of filepath -func extractBucket(fullpath util.FullPath) (string, string) { - if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") { - return "", string(fullpath) - } - if strings.Count(string(fullpath), "/") < 2 { - return "", string(fullpath) - } - bucketAndObjectKey := string(fullpath)[len("/buckets/"):] - t := strings.Index(bucketAndObjectKey, "/") - bucket := bucketAndObjectKey - shortPath := "/" - if t > 0 { - bucket = bucketAndObjectKey[:t] - shortPath = string(util.FullPath(bucketAndObjectKey[t:])) - } - return bucket, shortPath -} - // gets the collection the bucket points to from filepath func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) { bucket, _ := extractBucket(fullpath) @@ -80,6 +61,25 @@ func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpat return c, err } +// called by extractBucketCollection +func extractBucket(fullpath util.FullPath) (string, string) { + if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") { + return "", string(fullpath) + } + if strings.Count(string(fullpath), "/") < 3 { + return "", string(fullpath) + } + bucketAndObjectKey := string(fullpath)[len(BUCKET_PREFIX+"/"):] + t := strings.Index(bucketAndObjectKey, "/") + bucket := bucketAndObjectKey + shortPath := "/" + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = string(util.FullPath(bucketAndObjectKey[t:])) + } + return bucket, shortPath +} + // get bucket collection from cache. if not exist, creates the buckets collection and grab it func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) { var ok bool From 6701ac3a61a052a1f29b4a3fe852d53d7375845a Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:14:49 +0000 Subject: [PATCH 18/29] Merge branch 'chrislusf-master' into a --- go.mod | 2 +- go.sum | 20 +------------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 06e686e17..149b4894a 100644 --- a/go.mod +++ b/go.mod @@ -155,6 +155,7 @@ require ( ) require ( + github.com/arangodb/go-driver v1.2.1 github.com/fluent/fluent-logger-golang v1.8.0 github.com/hanwen/go-fuse/v2 v2.1.0 ) @@ -163,7 +164,6 @@ require ( cloud.google.com/go/compute v1.5.0 // indirect cloud.google.com/go/iam v0.1.1 // indirect github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect - github.com/arangodb/go-driver v1.2.1 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect diff --git a/go.sum b/go.sum index 15350f903..becd05248 100644 --- a/go.sum +++ b/go.sum @@ -178,15 +178,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -<<<<<<< HEAD -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8= github.com/arangodb/go-driver v1.2.1/go.mod h1:zdDkJJnCj8DAkfbtIjIXnsTrWIiy6VhP3Vy14p+uQeY= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= -======= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -283,11 +278,8 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -<<<<<<< HEAD github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= -======= github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -393,18 +385,11 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -<<<<<<< HEAD -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -======= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -793,14 +778,10 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -<<<<<<< HEAD github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -======= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seaweedfs/goexif v2.0.0+incompatible h1:x8pckiT12QQhifwhDQpeISgDfsqmQ6VR4LFPQ64JRps= @@ -1229,6 +1210,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 71307ce2e6ee06a39cf9c378ad27a548cb950e1b Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:25:53 +0000 Subject: [PATCH 19/29] test ci --- .gitlab-ci.yml | 25 ++ .vscode/configurationCache.log | 1 + .vscode/dryrun.log | 6 + .vscode/settings.json | 3 + .vscode/targets.log | 630 +++++++++++++++++++++++++++++++++ 5 files changed, 665 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 .vscode/configurationCache.log create mode 100644 .vscode/dryrun.log create mode 100644 .vscode/settings.json create mode 100644 .vscode/targets.log diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..9a17a8d7d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ + +build_binary_latest: + stage: build + image: + name: golang:1.18.0-alpine3.15 + script: + - go build -o ./weed/weed ./weed + rules: + - if: $CI_COMMIT_TAG + +build_container_latest: + stage: build + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + - >- + /kaniko/executor + --context "${CI_PROJECT_DIR}/docker" + --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" + rules: + - if: $CI_COMMIT_TAG \ No newline at end of file diff --git a/.vscode/configurationCache.log b/.vscode/configurationCache.log new file mode 100644 index 000000000..bab9054b3 --- /dev/null +++ b/.vscode/configurationCache.log @@ -0,0 +1 @@ +{"buildTargets":[],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}} \ No newline at end of file diff --git a/.vscode/dryrun.log b/.vscode/dryrun.log new file mode 100644 index 000000000..40a61ed1e --- /dev/null +++ b/.vscode/dryrun.log @@ -0,0 +1,6 @@ +make --dry-run --always-make --keep-going --print-directory +make: Entering directory '/home/coder/repos/seaweedfs' +make: Leaving directory '/home/coder/repos/seaweedfs' + +make: *** No targets specified and no makefile found. Stop. + diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..65e1ec078 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "makefile.extensionOutputFolder": "./.vscode" +} \ No newline at end of file diff --git a/.vscode/targets.log b/.vscode/targets.log new file mode 100644 index 000000000..6684e3315 --- /dev/null +++ b/.vscode/targets.log @@ -0,0 +1,630 @@ +make all --print-data-base --no-builtin-variables --no-builtin-rules --question +make: *** No rule to make target 'all'. Stop. + +# GNU Make 4.3 +# Built for x86_64-pc-linux-gnu +# Copyright (C) 1988-2020 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. + +# Make data base, printed on Mon Mar 28 17:22:08 2022 + +# Variables + +# environment +SEAWEED_VOLUME_1_SERVICE_PORT = 8444 +# environment +GETTY_CODE_CODE_SERVER_SERVICE_PORT = 8080 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PROTO = tcp +# environment +KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_CR_METRICS = 8686 +# environment +PLIK_SERVICE_HOST = 10.245.176.243 +# environment +SEAWEED_VOLUME_0_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +LC_ALL = C +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP = tcp://10.245.169.206:8686 +# environment +CODE_SERVER_SERVICE_HOST = 10.245.153.237 +# environment +PG_HA_HA_SERVICE_PORT = 5432 +# environment +SEAWEED_FILER_PORT_8333_TCP_PROTO = tcp +# environment +REDIS_REPLICAS_SERVICE_PORT_TCP_REDIS = 6379 +# environment +SEAWEED_FILER_PORT_8888_TCP = tcp://10.245.202.58:8888 +# environment +VSCODE_INJECT_NODE_MODULE_LOOKUP_PATH = /usr/lib/code-server/vendor/modules/code-oss-dev/remote/node_modules +# environment +REDIS_MASTER_SERVICE_HOST = 10.245.29.137 +# environment +REDIS_MASTER_PORT_6379_TCP = tcp://10.245.29.137:6379 +# environment +SEAWEED_FILER_PORT = tcp://10.245.202.58:8888 +# environment +REDIS_REPLICAS_PORT_6379_TCP = tcp://10.245.122.67:6379 +# environment +VSCODE_CWD = /home/coder +# environment +KUBERNETES_PORT_443_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_3_PORT_18444_TCP_ADDR = 10.245.205.95 +# environment +TRAEFIK_PORT_443_TCP = tcp://10.245.52.194:443 +# environment +SEAWEED_FILER_SERVICE_PORT_FILER_HTTP = 8888 +# environment +SEAWEED_VOLUME_2_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +PG_HA_REPLICAS_PORT_5432_TCP_PORT = 5432 +# environment +TRAEFIK_PORT_443_TCP_ADDR = 10.245.52.194 +# environment +CODE_SERVER_PARENT_PID = 12 +# default +MAKE_COMMAND := make +# environment +SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +SEAWEED_VOLUME_3_PORT_8444_TCP_PORT = 8444 +# environment +PASTEY_PORT_5000_TCP_PORT = 5000 +# environment +KUBERNETES_PORT_443_TCP_PORT = 443 +# environment +PG_HA_PGBOUNCER_PORT_5432_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_GRPC = 18444 +# environment +GOPATH = /home/coder/.asdf/installs/golang/1.18/packages +# automatic +@D = $(patsubst %/,%,$(dir $@)) +# environment +PG_HA_REPLICAS_SERVICE_PORT = 5432 +# environment +PG_HA_HA_PORT_5432_TCP_PORT = 5432 +# environment +GETTY_CODE_CODE_SERVER_SERVICE_HOST = 10.245.62.90 +# environment +PGADMIN4_SERVICE_HOST = 10.245.182.193 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_0_PORT = tcp://10.245.135.162:8444 +# environment +SEAWEED_FILER_PORT_8888_TCP_PROTO = tcp +# environment +SEAWEED_MASTER_PORT_19333_TCP = tcp://10.245.145.57:19333 +# environment +REDIS_MASTER_SERVICE_PORT = 6379 +# environment +SEAWEED_VOLUME_0_PORT_18444_TCP_PORT = 18444 +# environment +VSCODE_HANDLES_UNCAUGHT_ERRORS = true +# default +.VARIABLES := +# environment +PWD = /home/coder/repos/seaweedfs +# automatic +%D = $(patsubst %/,%,$(dir $%)) +# environment +HOSTNAME = code-server-5dfcd6ff77-w8kdz +# environment +PG_HA_PGBOUNCER_PORT_5432_TCP = tcp://10.245.186.4:5432 +# environment +SEAWEED_VOLUME_2_PORT_8444_TCP = tcp://10.245.184.0:8444 +# environment +PG_HA_HA_SERVICE_PORT_POSTGRES = 5432 +# environment +TRAEFIK_PORT_80_TCP_ADDR = 10.245.52.194 +# environment +PG_HA_REPLICAS_PORT = tcp://10.245.76.221:5432 +# environment +SEAWEED_FILER_PORT_18888_TCP = tcp://10.245.202.58:18888 +# automatic +^D = $(patsubst %/,%,$(dir $^)) +# environment +VSCODE_LOG_STACK = false +# automatic +%F = $(notdir $%) +# environment +PASTEY_SERVICE_PORT = 5000 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PORT = 8686 +# environment +LANG = C +# default +.LOADED := +# default +.INCLUDE_DIRS = /usr/local/include /usr/include /usr/include +# environment +PG_HA_HA_PORT_5432_TCP = tcp://10.245.23.155:5432 +# environment +TRAEFIK_PORT_22_TCP = tcp://10.245.52.194:22 +# makefile +MAKEFLAGS = pqrR +# environment +SEAWEED_VOLUME_1_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +REDIS_REPLICAS_PORT_6379_TCP_PROTO = tcp +# environment +TRAEFIK_SERVICE_PORT_WEBSECURE = 443 +# environment +REDIS_MASTER_SERVICE_PORT_TCP_REDIS = 6379 +# environment +SEAWEED_VOLUME_1_PORT_18444_TCP_PORT = 18444 +# environment +SEAWEED_VOLUME_2_SERVICE_PORT = 8444 +# makefile +CURDIR := /home/coder/repos/seaweedfs +# environment +SEAWEED_FILER_SERVICE_HOST = 10.245.202.58 +# environment +VSCODE_PIPE_LOGGING = true +# environment +SEAWEED_FILER_SERVICE_PORT_FILER_GRPC = 18888 +# environment +REDIS_REPLICAS_SERVICE_PORT = 6379 +# environment +PG_HA_PGBOUNCER_SERVICE_HOST = 10.245.186.4 +# environment +APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = true +# environment +PG_HA_REPLICAS_SERVICE_HOST = 10.245.76.221 +# automatic +*D = $(patsubst %/,%,$(dir $*)) +# environment +PGADMIN4_PORT_80_TCP_ADDR = 10.245.182.193 +# environment +SEAWEED_MASTER_PORT_9333_TCP = tcp://10.245.145.57:9333 +# environment +TRAEFIK_SERVICE_HOST = 10.245.52.194 +# default +.SHELLFLAGS := -c +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PORT = 8383 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_ADDR = 10.245.169.206 +# environment +MFLAGS = -pqrR +# automatic ++D = $(patsubst %/,%,$(dir $+)) +# environment +PASTEY_PORT_5000_TCP_PROTO = tcp +# makefile +MAKEFILE_LIST := +# environment +PG_HA_PGBOUNCER_SERVICE_PORT = 5432 +# environment +REDIS_MASTER_PORT_6379_TCP_ADDR = 10.245.29.137 +# environment +REDIS_REPLICAS_PORT = tcp://10.245.122.67:6379 +# environment +GETTY_CODE_CODE_SERVER_PORT_8080_TCP = tcp://10.245.62.90:8080 +# environment +PG_HA_REPLICAS_PORT_5432_TCP = tcp://10.245.76.221:5432 +# environment +SEAWEED_MASTER_PORT_9333_TCP_PROTO = tcp +# automatic +@F = $(notdir $@) +# environment +VSCODE_VERBOSE_LOGGING = true +# environment +TRAEFIK_PORT_443_TCP_PROTO = tcp +# automatic +?D = $(patsubst %/,%,$(dir $?)) +# environment +PASTEY_PORT = tcp://10.245.184.85:5000 +# environment +TRAEFIK_PORT_80_TCP_PROTO = tcp +# environment +SEAWEED_MASTER_PORT_9333_TCP_PORT = 9333 +# environment +GITEA_MEMCACHED_PORT = tcp://10.245.55.95:11211 +# environment +SEAWEED_VOLUME_0_PORT_18444_TCP_PROTO = tcp +# environment +SEAWEED_MASTER_PORT_19333_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_0_SERVICE_PORT = 8444 +# environment +PLIK_PORT_8080_TCP = tcp://10.245.176.243:8080 +# environment +KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_HTTP_METRICS = 8383 +# environment +SEAWEED_VOLUME_2_PORT_8444_TCP_ADDR = 10.245.184.0 +# environment +SEAWEED_VOLUME_2_PORT = tcp://10.245.184.0:8444 +# automatic +*F = $(notdir $*) +# environment +TRAEFIK_PORT_22_TCP_ADDR = 10.245.52.194 +# environment +KUBERNETES_PORT_443_TCP_ADDR = 10.245.0.1 +# environment +CODE_SERVER_PORT_8080_TCP = tcp://10.245.153.237:8080 +# environment +TRAEFIK_PORT_22_TCP_PORT = 22 +# automatic + Date: Mon, 28 Mar 2022 17:26:43 +0000 Subject: [PATCH 20/29] del --- .gitignore | 2 + .vscode/configurationCache.log | 1 - .vscode/dryrun.log | 6 - .vscode/settings.json | 3 - .vscode/targets.log | 630 --------------------------------- 5 files changed, 2 insertions(+), 640 deletions(-) delete mode 100644 .vscode/configurationCache.log delete mode 100644 .vscode/dryrun.log delete mode 100644 .vscode/settings.json delete mode 100644 .vscode/targets.log diff --git a/.gitignore b/.gitignore index e7ae92784..25a58bc67 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,8 @@ Temporary Items # Mongo Explorer plugin: # .idea/mongoSettings.xml +## vscode +.vscode ## File-based project format: *.ipr *.iws diff --git a/.vscode/configurationCache.log b/.vscode/configurationCache.log deleted file mode 100644 index bab9054b3..000000000 --- a/.vscode/configurationCache.log +++ /dev/null @@ -1 +0,0 @@ -{"buildTargets":[],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}} \ No newline at end of file diff --git a/.vscode/dryrun.log b/.vscode/dryrun.log deleted file mode 100644 index 40a61ed1e..000000000 --- a/.vscode/dryrun.log +++ /dev/null @@ -1,6 +0,0 @@ -make --dry-run --always-make --keep-going --print-directory -make: Entering directory '/home/coder/repos/seaweedfs' -make: Leaving directory '/home/coder/repos/seaweedfs' - -make: *** No targets specified and no makefile found. Stop. - diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 65e1ec078..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "makefile.extensionOutputFolder": "./.vscode" -} \ No newline at end of file diff --git a/.vscode/targets.log b/.vscode/targets.log deleted file mode 100644 index 6684e3315..000000000 --- a/.vscode/targets.log +++ /dev/null @@ -1,630 +0,0 @@ -make all --print-data-base --no-builtin-variables --no-builtin-rules --question -make: *** No rule to make target 'all'. Stop. - -# GNU Make 4.3 -# Built for x86_64-pc-linux-gnu -# Copyright (C) 1988-2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. - -# Make data base, printed on Mon Mar 28 17:22:08 2022 - -# Variables - -# environment -SEAWEED_VOLUME_1_SERVICE_PORT = 8444 -# environment -GETTY_CODE_CODE_SERVER_SERVICE_PORT = 8080 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PROTO = tcp -# environment -KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_CR_METRICS = 8686 -# environment -PLIK_SERVICE_HOST = 10.245.176.243 -# environment -SEAWEED_VOLUME_0_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -LC_ALL = C -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP = tcp://10.245.169.206:8686 -# environment -CODE_SERVER_SERVICE_HOST = 10.245.153.237 -# environment -PG_HA_HA_SERVICE_PORT = 5432 -# environment -SEAWEED_FILER_PORT_8333_TCP_PROTO = tcp -# environment -REDIS_REPLICAS_SERVICE_PORT_TCP_REDIS = 6379 -# environment -SEAWEED_FILER_PORT_8888_TCP = tcp://10.245.202.58:8888 -# environment -VSCODE_INJECT_NODE_MODULE_LOOKUP_PATH = /usr/lib/code-server/vendor/modules/code-oss-dev/remote/node_modules -# environment -REDIS_MASTER_SERVICE_HOST = 10.245.29.137 -# environment -REDIS_MASTER_PORT_6379_TCP = tcp://10.245.29.137:6379 -# environment -SEAWEED_FILER_PORT = tcp://10.245.202.58:8888 -# environment -REDIS_REPLICAS_PORT_6379_TCP = tcp://10.245.122.67:6379 -# environment -VSCODE_CWD = /home/coder -# environment -KUBERNETES_PORT_443_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_3_PORT_18444_TCP_ADDR = 10.245.205.95 -# environment -TRAEFIK_PORT_443_TCP = tcp://10.245.52.194:443 -# environment -SEAWEED_FILER_SERVICE_PORT_FILER_HTTP = 8888 -# environment -SEAWEED_VOLUME_2_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -PG_HA_REPLICAS_PORT_5432_TCP_PORT = 5432 -# environment -TRAEFIK_PORT_443_TCP_ADDR = 10.245.52.194 -# environment -CODE_SERVER_PARENT_PID = 12 -# default -MAKE_COMMAND := make -# environment -SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -SEAWEED_VOLUME_3_PORT_8444_TCP_PORT = 8444 -# environment -PASTEY_PORT_5000_TCP_PORT = 5000 -# environment -KUBERNETES_PORT_443_TCP_PORT = 443 -# environment -PG_HA_PGBOUNCER_PORT_5432_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_GRPC = 18444 -# environment -GOPATH = /home/coder/.asdf/installs/golang/1.18/packages -# automatic -@D = $(patsubst %/,%,$(dir $@)) -# environment -PG_HA_REPLICAS_SERVICE_PORT = 5432 -# environment -PG_HA_HA_PORT_5432_TCP_PORT = 5432 -# environment -GETTY_CODE_CODE_SERVER_SERVICE_HOST = 10.245.62.90 -# environment -PGADMIN4_SERVICE_HOST = 10.245.182.193 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_0_PORT = tcp://10.245.135.162:8444 -# environment -SEAWEED_FILER_PORT_8888_TCP_PROTO = tcp -# environment -SEAWEED_MASTER_PORT_19333_TCP = tcp://10.245.145.57:19333 -# environment -REDIS_MASTER_SERVICE_PORT = 6379 -# environment -SEAWEED_VOLUME_0_PORT_18444_TCP_PORT = 18444 -# environment -VSCODE_HANDLES_UNCAUGHT_ERRORS = true -# default -.VARIABLES := -# environment -PWD = /home/coder/repos/seaweedfs -# automatic -%D = $(patsubst %/,%,$(dir $%)) -# environment -HOSTNAME = code-server-5dfcd6ff77-w8kdz -# environment -PG_HA_PGBOUNCER_PORT_5432_TCP = tcp://10.245.186.4:5432 -# environment -SEAWEED_VOLUME_2_PORT_8444_TCP = tcp://10.245.184.0:8444 -# environment -PG_HA_HA_SERVICE_PORT_POSTGRES = 5432 -# environment -TRAEFIK_PORT_80_TCP_ADDR = 10.245.52.194 -# environment -PG_HA_REPLICAS_PORT = tcp://10.245.76.221:5432 -# environment -SEAWEED_FILER_PORT_18888_TCP = tcp://10.245.202.58:18888 -# automatic -^D = $(patsubst %/,%,$(dir $^)) -# environment -VSCODE_LOG_STACK = false -# automatic -%F = $(notdir $%) -# environment -PASTEY_SERVICE_PORT = 5000 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PORT = 8686 -# environment -LANG = C -# default -.LOADED := -# default -.INCLUDE_DIRS = /usr/local/include /usr/include /usr/include -# environment -PG_HA_HA_PORT_5432_TCP = tcp://10.245.23.155:5432 -# environment -TRAEFIK_PORT_22_TCP = tcp://10.245.52.194:22 -# makefile -MAKEFLAGS = pqrR -# environment -SEAWEED_VOLUME_1_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -REDIS_REPLICAS_PORT_6379_TCP_PROTO = tcp -# environment -TRAEFIK_SERVICE_PORT_WEBSECURE = 443 -# environment -REDIS_MASTER_SERVICE_PORT_TCP_REDIS = 6379 -# environment -SEAWEED_VOLUME_1_PORT_18444_TCP_PORT = 18444 -# environment -SEAWEED_VOLUME_2_SERVICE_PORT = 8444 -# makefile -CURDIR := /home/coder/repos/seaweedfs -# environment -SEAWEED_FILER_SERVICE_HOST = 10.245.202.58 -# environment -VSCODE_PIPE_LOGGING = true -# environment -SEAWEED_FILER_SERVICE_PORT_FILER_GRPC = 18888 -# environment -REDIS_REPLICAS_SERVICE_PORT = 6379 -# environment -PG_HA_PGBOUNCER_SERVICE_HOST = 10.245.186.4 -# environment -APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = true -# environment -PG_HA_REPLICAS_SERVICE_HOST = 10.245.76.221 -# automatic -*D = $(patsubst %/,%,$(dir $*)) -# environment -PGADMIN4_PORT_80_TCP_ADDR = 10.245.182.193 -# environment -SEAWEED_MASTER_PORT_9333_TCP = tcp://10.245.145.57:9333 -# environment -TRAEFIK_SERVICE_HOST = 10.245.52.194 -# default -.SHELLFLAGS := -c -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PORT = 8383 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_ADDR = 10.245.169.206 -# environment -MFLAGS = -pqrR -# automatic -+D = $(patsubst %/,%,$(dir $+)) -# environment -PASTEY_PORT_5000_TCP_PROTO = tcp -# makefile -MAKEFILE_LIST := -# environment -PG_HA_PGBOUNCER_SERVICE_PORT = 5432 -# environment -REDIS_MASTER_PORT_6379_TCP_ADDR = 10.245.29.137 -# environment -REDIS_REPLICAS_PORT = tcp://10.245.122.67:6379 -# environment -GETTY_CODE_CODE_SERVER_PORT_8080_TCP = tcp://10.245.62.90:8080 -# environment -PG_HA_REPLICAS_PORT_5432_TCP = tcp://10.245.76.221:5432 -# environment -SEAWEED_MASTER_PORT_9333_TCP_PROTO = tcp -# automatic -@F = $(notdir $@) -# environment -VSCODE_VERBOSE_LOGGING = true -# environment -TRAEFIK_PORT_443_TCP_PROTO = tcp -# automatic -?D = $(patsubst %/,%,$(dir $?)) -# environment -PASTEY_PORT = tcp://10.245.184.85:5000 -# environment -TRAEFIK_PORT_80_TCP_PROTO = tcp -# environment -SEAWEED_MASTER_PORT_9333_TCP_PORT = 9333 -# environment -GITEA_MEMCACHED_PORT = tcp://10.245.55.95:11211 -# environment -SEAWEED_VOLUME_0_PORT_18444_TCP_PROTO = tcp -# environment -SEAWEED_MASTER_PORT_19333_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_0_SERVICE_PORT = 8444 -# environment -PLIK_PORT_8080_TCP = tcp://10.245.176.243:8080 -# environment -KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_HTTP_METRICS = 8383 -# environment -SEAWEED_VOLUME_2_PORT_8444_TCP_ADDR = 10.245.184.0 -# environment -SEAWEED_VOLUME_2_PORT = tcp://10.245.184.0:8444 -# automatic -*F = $(notdir $*) -# environment -TRAEFIK_PORT_22_TCP_ADDR = 10.245.52.194 -# environment -KUBERNETES_PORT_443_TCP_ADDR = 10.245.0.1 -# environment -CODE_SERVER_PORT_8080_TCP = tcp://10.245.153.237:8080 -# environment -TRAEFIK_PORT_22_TCP_PORT = 22 -# automatic - Date: Mon, 28 Mar 2022 17:28:53 +0000 Subject: [PATCH 21/29] Replace with hyphen --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a17a8d7d..29f8fd342 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,4 @@ - -build_binary_latest: +build-binary-latest: stage: build image: name: golang:1.18.0-alpine3.15 @@ -8,7 +7,7 @@ build_binary_latest: rules: - if: $CI_COMMIT_TAG -build_container_latest: +build-container-latest: stage: build image: name: gcr.io/kaniko-project/executor:debug From 9f0957b4d09feffd4c587b0da4257b83e16acc52 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:42:03 +0000 Subject: [PATCH 22/29] Remove restrictions --- .gitlab-ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29f8fd342..608ab54fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,8 +4,6 @@ build-binary-latest: name: golang:1.18.0-alpine3.15 script: - go build -o ./weed/weed ./weed - rules: - - if: $CI_COMMIT_TAG build-container-latest: stage: build @@ -19,6 +17,4 @@ build-container-latest: /kaniko/executor --context "${CI_PROJECT_DIR}/docker" --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" - rules: - - if: $CI_COMMIT_TAG \ No newline at end of file + --destination "${CI_REGISTRY_IMAGE}:latest" \ No newline at end of file From c464e2e0d5d2ab0be9c6a269b1ef4f848d085803 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:43:03 +0000 Subject: [PATCH 23/29] Remove restrictions --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 608ab54fa..58112db6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,4 +17,5 @@ build-container-latest: /kaniko/executor --context "${CI_PROJECT_DIR}/docker" --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" - --destination "${CI_REGISTRY_IMAGE}:latest" \ No newline at end of file + --destination "${CI_REGISTRY_IMAGE}:latest" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" \ No newline at end of file From ca6becfeec7ca94736d5e8b46e78b5564dd44fe8 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:08:16 +0000 Subject: [PATCH 24/29] df changes --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58112db6c..65d88908e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,11 @@ build-binary-latest: image: name: golang:1.18.0-alpine3.15 script: - - go build -o ./weed/weed ./weed + - go build -o ./weed/weed ./weed + - cp ./weed/weed docker/weed + artifacts: + paths: + - ./weed/weed build-container-latest: stage: build @@ -16,6 +20,6 @@ build-container-latest: - >- /kaniko/executor --context "${CI_PROJECT_DIR}/docker" - --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" + --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.local" --destination "${CI_REGISTRY_IMAGE}:latest" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" \ No newline at end of file From 644a6139868393ce5acee134da8b43dc4c48c1f8 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:15:47 +0000 Subject: [PATCH 25/29] needs --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65d88908e..4d5b9674d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ build-container-latest: image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] + needs: ["build-binary-latest"] script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json From ec6320b037894b4cc023d0c5f078a2cba5c5fabe Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:16:24 +0000 Subject: [PATCH 26/29] Add gitg --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d5b9674d..d42960b33 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ build-binary-latest: image: name: golang:1.18.0-alpine3.15 script: + - apk add git - go build -o ./weed/weed ./weed - cp ./weed/weed docker/weed artifacts: From 1d47574531b8475751eb0338dd10f30a1e16d1ff Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:22:38 +0000 Subject: [PATCH 27/29] add build base --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d42960b33..3bbf60537 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ build-binary-latest: image: name: golang:1.18.0-alpine3.15 script: - - apk add git + - apk add git build-base - go build -o ./weed/weed ./weed - cp ./weed/weed docker/weed artifacts: From ea0b41d91c0f017cf4a30bba83f88b94197f38fc Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 11:24:24 -0700 Subject: [PATCH 28/29] remove --- .gitlab-ci.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 3bbf60537..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,27 +0,0 @@ -build-binary-latest: - stage: build - image: - name: golang:1.18.0-alpine3.15 - script: - - apk add git build-base - - go build -o ./weed/weed ./weed - - cp ./weed/weed docker/weed - artifacts: - paths: - - ./weed/weed - -build-container-latest: - stage: build - image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] - needs: ["build-binary-latest"] - script: - - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json - - >- - /kaniko/executor - --context "${CI_PROJECT_DIR}/docker" - --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.local" - --destination "${CI_REGISTRY_IMAGE}:latest" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" \ No newline at end of file From 7e925175715c57b49552a94d25d1d3dc40f1d881 Mon Sep 17 00:00:00 2001 From: a Date: Fri, 1 Apr 2022 14:09:25 -0500 Subject: [PATCH 29/29] change user and pass to username and password --- weed/command/scaffold/filer.toml | 4 ++-- weed/filer/arangodb/arangodb_store.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 6a7835de3..0a505bbdc 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -290,8 +290,8 @@ enabled = false db_name = "seaweedfs" servers=["http://localhost:8529"] # list of servers to connect to # only basic auth supported for now -user="" -pass="" +username="" +password="" # skip tls cert validation insecure_skip_verify = true diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 27ed9132b..9fd1fffb3 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -59,8 +59,8 @@ func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix store.buckets = make(map[string]driver.Collection, 3) store.databaseName = configuration.GetString(prefix + "db_name") return store.connection(configuration.GetStringSlice(prefix+"servers"), - configuration.GetString(prefix+"user"), - configuration.GetString(prefix+"pass"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), configuration.GetBool(prefix+"insecure_skip_verify"), ) }