From 68cf6a922954587d0457a100a7576afc209f26eb Mon Sep 17 00:00:00 2001 From: aaaaa Date: Thu, 17 Mar 2022 06:07:41 +0000 Subject: [PATCH 01/50] a --- .gitpod.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 000000000..fe8e4d9d9 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,10 @@ +tasks: + - init: go mod tidy + - name: redis + command: docker run -p 6379:6379 redis:6.2.6 + - name: arangodb + command: docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0 + - name: postgresql + command: docker run -p 5432:5432 -e POSTGRES_PASSWORD=test postgres:14.2 + - name: cassandra + command: docker run -p 7000:7000 cassandra:4.0.3 From 921535001a62424791549d51f6c91f3baefd2f1b Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 04:49:26 -0500 Subject: [PATCH 02/50] arangodb adapter --- .gitignore | 2 + go.mod | 4 +- go.sum | 11 + weed/command/imports.go | 1 + weed/command/scaffold/filer.toml | 6 + weed/filer/arangodb/arangodb_store.go | 348 +++++++++++++++++++++++ weed/filer/arangodb/arangodb_store_kv.go | 62 ++++ weed/server/filer_server.go | 1 + 8 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 weed/filer/arangodb/arangodb_store.go create mode 100644 weed/filer/arangodb/arangodb_store_kv.go diff --git a/.gitignore b/.gitignore index 671b01051..e7ae92784 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,8 @@ com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties +workspace/ + test_data build target diff --git a/go.mod b/go.mod index 1507a16ea..862cf7ecb 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/go-stack/stack v1.8.0 // indirect github.com/go-zookeeper/zk v1.0.2 // indirect github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d - github.com/golang-jwt/jwt v3.2.1+incompatible + github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 // indirect @@ -169,6 +169,8 @@ require ( require ( cloud.google.com/go/kms v1.0.0 // indirect github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect + github.com/arangodb/go-driver v1.2.1 // indirect + github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/d4l3k/messagediff v1.2.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 7d3b72eb6..7bec2c973 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,10 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8= +github.com/arangodb/go-driver v1.2.1/go.mod h1:zdDkJJnCj8DAkfbtIjIXnsTrWIiy6VhP3Vy14p+uQeY= +github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= +github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -176,6 +180,7 @@ github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636Fk github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -193,6 +198,7 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -290,6 +296,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -690,6 +698,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -1092,6 +1102,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/weed/command/imports.go b/weed/command/imports.go index 3792c45c4..5b3195907 100644 --- a/weed/command/imports.go +++ b/weed/command/imports.go @@ -15,6 +15,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" _ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" + _ "github.com/chrislusf/seaweedfs/weed/filer/arangodb" _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 5d4513c36..19da66dc1 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -285,6 +285,12 @@ healthcheck_enabled = false index.max_result_window = 10000 +[arangodb] # in development dont use it +enabled = false +arango_host=["http://localhost:8529"] +arango_user="" +arango_pass="" + ########################## ########################## diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go new file mode 100644 index 000000000..3cc9e14c9 --- /dev/null +++ b/weed/filer/arangodb/arangodb_store.go @@ -0,0 +1,348 @@ +package arangodb + +import ( + "context" + "crypto/md5" + "crypto/tls" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "time" + + "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func init() { + filer.Stores = append(filer.Stores, &ArangodbStore{}) +} + +type ArangodbStore struct { + connect driver.Connection + client driver.Client + database driver.Database + collection driver.Collection +} + +type Model struct { + Key string `json:"_key"` + Directory string `json:"directory"` + Name string `json:"name"` + Meta []uint64 `json:"meta"` +} + +func (store *ArangodbStore) GetName() string { + return "arangodb" +} + +func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.connection(configuration.GetStringSlice(prefix+"arango_host"), + configuration.GetString(prefix+"arango_user"), + configuration.GetString(prefix+"arango_pass"), + ) +} + +func (store *ArangodbStore) connection(uris []string, user string, pass string) (err error) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + + store.connect, err = http.NewConnection(http.ConnectionConfig{ + Endpoints: uris, + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }) + if err != nil { + return err + } + store.client, err = driver.NewClient(driver.ClientConfig{ + Connection: store.connect, + Authentication: driver.BasicAuthentication(user, pass), + }) + if err != nil { + return err + } + db_name := "seaweed-filer" + ok, err := store.client.DatabaseExists(ctx, db_name) + if err != nil { + return err + } + if ok { + store.database, err = store.client.Database(ctx, db_name) + } else { + store.database, err = store.client.CreateDatabase(ctx, db_name, &driver.CreateDatabaseOptions{}) + } + if err != nil { + return err + } + + coll_name := "files" + ok, err = store.database.CollectionExists(ctx, coll_name) + if err != nil { + return err + } + if ok { + store.collection, err = store.database.Collection(ctx, coll_name) + } else { + store.collection, err = store.database.CreateCollection(ctx, coll_name, &driver.CreateCollectionOptions{}) + } + if err != nil { + return err + } + + // ensure indices + + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, &driver.EnsurePersistentIndexOptions{ + Name: "directory_name_multi", + Unique: true, + }); err != nil { + return err + } + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, + &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { + return err + } + + if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory_fulltext"}, + &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { + return err + } + + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ + Name: "IDX_name", + }); err != nil { + return err + } + + return err +} + +type key int + +const ( + transactionKey key = 0 +) + +func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) { + txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{ + Exclusive: []string{"files"}, + }, &driver.BeginTransactionOptions{}) + if err != nil { + return nil, err + } + + return context.WithValue(ctx, transactionKey, txn), nil +} + +func (store *ArangodbStore) CommitTransaction(ctx context.Context) error { + val := ctx.Value(transactionKey) + cast, ok := val.(driver.TransactionID) + if !ok { + return fmt.Errorf("txn cast fail %s:", val) + } + err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{}) + if err != nil { + return err + } + return nil +} + +func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error { + val := ctx.Value(transactionKey) + cast, ok := val.(driver.TransactionID) + if !ok { + return fmt.Errorf("txn cast fail %s:", val) + } + err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{}) + if err != nil { + return err + } + return nil +} + +func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if len(entry.Chunks) > 50 { + meta = util.MaybeGzipData(meta) + } + model := &Model{ + Key: hashString(string(entry.FullPath)), + Directory: dir, + Name: name, + Meta: bytesToArray(meta), + } + _, err = store.collection.CreateDocument(ctx, model) + + if err != nil { + return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) + } + + return nil + +} + +func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if len(entry.Chunks) > 50 { + meta = util.MaybeGzipData(meta) + } + model := &Model{ + Key: hashString(string(entry.FullPath)), + Directory: dir, + Name: name, + Meta: bytesToArray(meta), + } + + _, err = store.collection.UpdateDocument(ctx, model.Key, model) + + if err != nil { + return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) + } + + return nil +} + +func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + var data Model + _, err = store.collection.ReadDocument(ctx, hashString(string(fullpath)), &data) + if driver.IsNotFound(err) { + return nil, filer_pb.ErrNotFound + } + if err != nil { + glog.Errorf("find %s: %v", fullpath, err) + return nil, filer_pb.ErrNotFound + } + if len(data.Meta) == 0 { + return nil, filer_pb.ErrNotFound + } + entry = &filer.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta))) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { + _, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath))) + if err != nil { + glog.Errorf("find %s: %v", fullpath, err) + return fmt.Errorf("delete %s : %v", fullpath, err) + } + + return nil +} + +func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { + dir, _ := fullpath.DirAndName() + cur, err := store.database.Query(ctx, ` +for d in files +filter d.directory == @dir +remove d in files`, map[string]interface{}{"dir": dir}) + if err != nil { + return fmt.Errorf("delete %s : %v", fullpath, err) + } + defer cur.Close() + return nil +} + +func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed +} + +func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + dir, name := dirPath.DirAndName() + eq := "" + if includeStartFile { + eq = "=" + } + _ = eq + _ = name + + cur, err := store.database.Query(ctx, fmt.Sprintf(` +for d in files +filter d.directory == @dir +sort d.name desc +limit %d +return d`, limit), map[string]interface{}{ + "dir": dir, + }) + if err != nil { + return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) + } + defer cur.Close() + for cur.HasMore() { + var data Model + _, err = cur.ReadDocument(ctx, &data) + if err != nil { + break + } + entry := &filer.Entry{ + FullPath: util.NewFullPath(string(dirPath), data.Name), + } + lastFileName = data.Name + converted := arrayToBytes(data.Meta) + if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } + + if !eachEntryFunc(entry) { + break + } + + } + return lastFileName, err +} + +func (store *ArangodbStore) Shutdown() { +} + +func hashString(dir string) string { + h := md5.New() + io.WriteString(h, dir) + b := h.Sum(nil) + return hex.EncodeToString(b) +} + +func bytesToArray(bs []byte) []uint64 { + out := make([]uint64, 0, 2+len(bs)/8) + out = append(out, uint64(len(bs))) + for len(bs)%8 != 0 { + bs = append(bs, 0) + } + for i := 0; i < len(bs); i = i + 8 { + out = append(out, binary.BigEndian.Uint64(bs[i:])) + } + return out +} + +func arrayToBytes(xs []uint64) []byte { + if len(xs) < 2 { + return []byte{} + } + first := xs[0] + out := make([]byte, len(xs)*8) + for i := 1; i < len(xs); i = i + 1 { + binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i]) + } + return out[:first] +} diff --git a/weed/filer/arangodb/arangodb_store_kv.go b/weed/filer/arangodb/arangodb_store_kv.go new file mode 100644 index 000000000..93caa75ed --- /dev/null +++ b/weed/filer/arangodb/arangodb_store_kv.go @@ -0,0 +1,62 @@ +package arangodb + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + dir, name := genDirAndName(key) + model := &Model{ + Key: hashString(string(key)), + Directory: dir, + Name: name, + Meta: bytesToArray(value), + } + + exists, err := store.collection.DocumentExists(ctx, model.Key) + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + if exists { + _, err = store.collection.UpdateDocument(ctx, model.Key, model) + } else { + _, err = store.collection.CreateDocument(ctx, model) + } + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + var model Model + _, err = store.collection.ReadDocument(ctx, hashString(string(key)), &model) + if err != nil { + glog.Errorf("kv get: %v", err) + return nil, filer.ErrKvNotFound + } + return arrayToBytes(model.Meta), nil +} + +func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) { + _, err = store.collection.RemoveDocument(ctx, hashString(string(key))) + if err != nil { + glog.Errorf("kv del: %v", err) + return filer.ErrKvNotFound + } + return nil +} + +func genDirAndName(key []byte) (dir string, name string) { + for len(key) < 8 { + key = append(key, 0) + } + dir = string(key[:8]) + name = string(key[8:]) + return +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 497f59568..e25de4795 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -21,6 +21,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer" + _ "github.com/chrislusf/seaweedfs/weed/filer/arangodb" _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" From 31571dd96e9c3b299dc72d0afac2859098248fc9 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 05:07:40 -0500 Subject: [PATCH 03/50] fix file listing --- weed/filer/arangodb/arangodb_store.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 3cc9e14c9..dc4fbad84 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -268,22 +268,19 @@ func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, di } func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - dir, name := dirPath.DirAndName() eq := "" - if includeStartFile { - eq = "=" + if !includeStartFile { + eq = "filter d.name != \"" + startFileName + "\"" } - _ = eq - _ = name - - cur, err := store.database.Query(ctx, fmt.Sprintf(` + fmt.Println(dirPath, startFileName, includeStartFile) + query := fmt.Sprintf(` for d in files -filter d.directory == @dir +filter d.directory == "%s" sort d.name desc +%s limit %d -return d`, limit), map[string]interface{}{ - "dir": dir, - }) +return d`, string(dirPath), eq, limit) + cur, err := store.database.Query(ctx, query, nil) if err != nil { return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) } From f3ab6769e991554969fde3f29fe170a38d34a996 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 05:09:13 -0500 Subject: [PATCH 04/50] make start from indexing work --- weed/filer/arangodb/arangodb_store.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index dc4fbad84..f10576105 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -263,16 +263,18 @@ remove d in files`, map[string]interface{}{"dir": dir}) return nil } +//TODO: use fulltext index func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { eq := "" - if !includeStartFile { - eq = "filter d.name != \"" + startFileName + "\"" + if includeStartFile { + eq = "filter d.name >= \"" + startFileName + "\"" + } else { + eq = "filter d.name > \"" + startFileName + "\"" } - fmt.Println(dirPath, startFileName, includeStartFile) query := fmt.Sprintf(` for d in files filter d.directory == "%s" From 94883e2fadefae5142bcfa5420e38143d0725ada Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 05:23:12 -0500 Subject: [PATCH 05/50] ok then --- weed/filer/arangodb/arangodb_store.go | 46 +++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index f10576105..7d47ce2d5 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -107,7 +107,7 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) return err } - if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory_fulltext"}, + if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { return err } @@ -263,11 +263,53 @@ remove d in files`, map[string]interface{}{"dir": dir}) return nil } -//TODO: use fulltext index func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } +//func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { +// eq := "" +// if includeStartFile { +// eq = "filter d.name >= \"" + startFileName + "\"" +// } else { +// eq = "filter d.name > \"" + startFileName + "\"" +// } +// query := fmt.Sprintf(` +//for d in fulltext(files,"directory","prefix:%s") +//sort d.name desc +//%s +//limit %d +//return d`, string(dirPath), eq, limit) +// cur, err := store.database.Query(ctx, query, nil) +// if err != nil { +// return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) +// } +// defer cur.Close() +// for cur.HasMore() { +// var data Model +// _, err = cur.ReadDocument(ctx, &data) +// if err != nil { +// break +// } +// entry := &filer.Entry{ +// FullPath: util.NewFullPath(data.Directory, data.Name), +// } +// lastFileName = data.Name +// converted := arrayToBytes(data.Meta) +// if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { +// err = decodeErr +// glog.V(0).Infof("list %s : %v", entry.FullPath, err) +// break +// } +// +// if !eachEntryFunc(entry) { +// break +// } +// +// } +// return lastFileName, err +//} + func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { eq := "" if includeStartFile { From 52ce8cc4f27f8a575147402244e2c084ef58a241 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 14:06:11 -0500 Subject: [PATCH 06/50] remove gitpod yml --- .gitpod.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml deleted file mode 100644 index fe8e4d9d9..000000000 --- a/.gitpod.yml +++ /dev/null @@ -1,10 +0,0 @@ -tasks: - - init: go mod tidy - - name: redis - command: docker run -p 6379:6379 redis:6.2.6 - - name: arangodb - command: docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0 - - name: postgresql - command: docker run -p 5432:5432 -e POSTGRES_PASSWORD=test postgres:14.2 - - name: cassandra - command: docker run -p 7000:7000 cassandra:4.0.3 From a0df993cefd4872430097cbe598ab91b9817aea3 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 14:08:36 -0500 Subject: [PATCH 07/50] cleanup, add a few comments --- weed/filer/arangodb/arangodb_store.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 7d47ce2d5..9095a96f9 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -106,11 +106,12 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } - - if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, - &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { - return err - } + // fulltext index not required since no prefix search + // user should just make one themselves if they intend on using it + // if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, + // &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { + // return err + // } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ Name: "IDX_name", @@ -267,6 +268,8 @@ func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, di return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed } +//TODO: i must be misunderstanding what this function is supposed to do +//so figure it out is the todo, i guess lol - aaaaa //func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { // eq := "" // if includeStartFile { @@ -357,6 +360,7 @@ return d`, string(dirPath), eq, limit) func (store *ArangodbStore) Shutdown() { } +//convert a string into arango-key safe hex bytes hash func hashString(dir string) string { h := md5.New() io.WriteString(h, dir) From bf745bdccb3d35c661cf75c10534ece0c0d9d392 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 14:09:34 -0500 Subject: [PATCH 08/50] revise comment --- weed/filer/arangodb/arangodb_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 9095a96f9..e0f036344 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -106,8 +106,8 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } - // fulltext index not required since no prefix search - // user should just make one themselves if they intend on using it + // fulltext index not required since no prefix search + // might change // if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, // &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { // return err From 423ce57cde83433103a4a59c04421c8c33b84287 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 21:12:25 -0500 Subject: [PATCH 09/50] prefix search, bucket implemented --- weed/command/scaffold/filer.toml | 10 +- weed/filer/arangodb/arangodb_store.go | 224 +++++++++---------- weed/filer/arangodb/arangodb_store_bucket.go | 30 +++ weed/filer/arangodb/arangodb_store_kv.go | 26 +-- weed/filer/arangodb/helpers.go | 44 ++++ weed/filer/arangodb/readme.md | 29 +++ weed/filer/filer_delete_entry.go | 2 - 7 files changed, 227 insertions(+), 138 deletions(-) create mode 100644 weed/filer/arangodb/arangodb_store_bucket.go create mode 100644 weed/filer/arangodb/helpers.go create mode 100644 weed/filer/arangodb/readme.md diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 19da66dc1..6a7835de3 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -287,9 +287,13 @@ index.max_result_window = 10000 [arangodb] # in development dont use it enabled = false -arango_host=["http://localhost:8529"] -arango_user="" -arango_pass="" +db_name = "seaweedfs" +servers=["http://localhost:8529"] # list of servers to connect to +# only basic auth supported for now +user="" +pass="" +# skip tls cert validation +insecure_skip_verify = true ########################## diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index e0f036344..a9f55f5bc 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -2,12 +2,9 @@ package arangodb import ( "context" - "crypto/md5" "crypto/tls" - "encoding/binary" - "encoding/hex" "fmt" - "io" + "strings" "time" "github.com/arangodb/go-driver" @@ -27,13 +24,20 @@ type ArangodbStore struct { client driver.Client database driver.Database collection driver.Collection + + databaseName string } type Model struct { - Key string `json:"_key"` - Directory string `json:"directory"` - Name string `json:"name"` - Meta []uint64 `json:"meta"` + Key string `json:"_key"` + Directory string `json:"directory,omitempty"` + Name string `json:"name,omitempty"` + Bucket string `json:"bucket,omitempty"` + + //arangodb does not support binary blobs + //we encode byte slice into uint64 slice + //see helpers.go + Meta []uint64 `json:"meta"` } func (store *ArangodbStore) GetName() string { @@ -41,19 +45,21 @@ func (store *ArangodbStore) GetName() string { } func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) { - return store.connection(configuration.GetStringSlice(prefix+"arango_host"), - configuration.GetString(prefix+"arango_user"), - configuration.GetString(prefix+"arango_pass"), + store.databaseName = configuration.GetString(prefix + "db_name") + return store.connection(configuration.GetStringSlice(prefix+"servers"), + configuration.GetString(prefix+"user"), + configuration.GetString(prefix+"pass"), + configuration.GetBool(prefix+"insecure_skip_verify"), ) } -func (store *ArangodbStore) connection(uris []string, user string, pass string) (err error) { +func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) { ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) store.connect, err = http.NewConnection(http.ConnectionConfig{ Endpoints: uris, TLSConfig: &tls.Config{ - InsecureSkipVerify: true, + InsecureSkipVerify: insecure, }, }) if err != nil { @@ -66,15 +72,14 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) if err != nil { return err } - db_name := "seaweed-filer" - ok, err := store.client.DatabaseExists(ctx, db_name) + ok, err := store.client.DatabaseExists(ctx, store.databaseName) if err != nil { return err } if ok { - store.database, err = store.client.Database(ctx, db_name) + store.database, err = store.client.Database(ctx, store.databaseName) } else { - store.database, err = store.client.CreateDatabase(ctx, db_name, &driver.CreateDatabaseOptions{}) + store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{}) } if err != nil { return err @@ -96,29 +101,29 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string) // ensure indices - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, &driver.EnsurePersistentIndexOptions{ - Name: "directory_name_multi", - Unique: true, - }); err != nil { + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, + &driver.EnsurePersistentIndexOptions{ + Name: "directory_name_multi", + Unique: true, + }); err != nil { return err } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } - // fulltext index not required since no prefix search - // might change - // if _, _, err = store.collection.EnsureFullTextIndex(ctx, []string{"directory"}, - // &driver.EnsureFullTextIndexOptions{Name: "IDX_FULLTEXT_directory", MinLength: 1}); err != nil { - // return err - // } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ Name: "IDX_name", }); err != nil { return err } - + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"bucket"}, &driver.EnsurePersistentIndexOptions{ + Name: "IDX_bucket", + Sparse: true, //sparse index, to locate files of bucket + }); err != nil { + return err + } return err } @@ -175,22 +180,41 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) if len(entry.Chunks) > 50 { meta = util.MaybeGzipData(meta) } + bucket, _ := extractBucket(entry.FullPath) model := &Model{ Key: hashString(string(entry.FullPath)), Directory: dir, Name: name, Meta: bytesToArray(meta), + Bucket: bucket, } _, err = store.collection.CreateDocument(ctx, model) - + if driver.IsConflict(err) { + return store.UpdateEntry(ctx, entry) + } if err != nil { - return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) + return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err) } return nil } +func extractBucket(fullpath util.FullPath) (string, string) { + if !strings.HasPrefix(string(fullpath), "/buckets/") { + return "", string(fullpath) + } + bucketAndObjectKey := string(fullpath)[len("/buckets/"):] + t := strings.Index(bucketAndObjectKey, "/") + bucket := bucketAndObjectKey + shortPath := "/" + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = string(util.FullPath(bucketAndObjectKey[t:])) + } + return bucket, shortPath +} + func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -243,20 +267,20 @@ func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPat func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { _, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath))) - if err != nil { + if err != nil && !driver.IsNotFound(err) { glog.Errorf("find %s: %v", fullpath, err) return fmt.Errorf("delete %s : %v", fullpath, err) } - return nil } func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { - dir, _ := fullpath.DirAndName() - cur, err := store.database.Query(ctx, ` -for d in files -filter d.directory == @dir -remove d in files`, map[string]interface{}{"dir": dir}) + query := "" + query = fmt.Sprintf(`for d in files filter starts_with(d.directory, "%s") remove d._key in files`, + strings.Join(strings.Split(string(fullpath), "/"), ","), + string(fullpath), + ) + cur, err := store.database.Query(ctx, query, nil) if err != nil { return fmt.Errorf("delete %s : %v", fullpath, err) } @@ -265,53 +289,53 @@ remove d in files`, map[string]interface{}{"dir": dir}) } func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed -} + // if no prefix, then dont use index + if prefix == "" { + return store.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) + } + eq := "" + if includeStartFile { + eq = "filter d.name >= \"" + startFileName + "\"" + } else { + eq = "filter d.name > \"" + startFileName + "\"" + } + query := fmt.Sprintf(` +for d in files +filter d.directory == @dir +filter starts_with(d.name, @prefix) +%s +sort d.name asc +limit %d +return d`, eq, limit) + cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath, "prefix": prefix}) + if err != nil { + return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) + } + defer cur.Close() + for cur.HasMore() { + var data Model + _, err = cur.ReadDocument(ctx, &data) + if err != nil { + break + } + entry := &filer.Entry{ + FullPath: util.NewFullPath(data.Directory, data.Name), + } + lastFileName = data.Name + converted := arrayToBytes(data.Meta) + if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } -//TODO: i must be misunderstanding what this function is supposed to do -//so figure it out is the todo, i guess lol - aaaaa -//func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { -// eq := "" -// if includeStartFile { -// eq = "filter d.name >= \"" + startFileName + "\"" -// } else { -// eq = "filter d.name > \"" + startFileName + "\"" -// } -// query := fmt.Sprintf(` -//for d in fulltext(files,"directory","prefix:%s") -//sort d.name desc -//%s -//limit %d -//return d`, string(dirPath), eq, limit) -// cur, err := store.database.Query(ctx, query, nil) -// if err != nil { -// return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) -// } -// defer cur.Close() -// for cur.HasMore() { -// var data Model -// _, err = cur.ReadDocument(ctx, &data) -// if err != nil { -// break -// } -// entry := &filer.Entry{ -// FullPath: util.NewFullPath(data.Directory, data.Name), -// } -// lastFileName = data.Name -// converted := arrayToBytes(data.Meta) -// if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { -// err = decodeErr -// glog.V(0).Infof("list %s : %v", entry.FullPath, err) -// break -// } -// -// if !eachEntryFunc(entry) { -// break -// } -// -// } -// return lastFileName, err -//} + if !eachEntryFunc(entry) { + break + } + + } + return lastFileName, err +} func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { eq := "" @@ -323,8 +347,8 @@ func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath ut query := fmt.Sprintf(` for d in files filter d.directory == "%s" -sort d.name desc %s +sort d.name asc limit %d return d`, string(dirPath), eq, limit) cur, err := store.database.Query(ctx, query, nil) @@ -359,35 +383,3 @@ return d`, string(dirPath), eq, limit) func (store *ArangodbStore) Shutdown() { } - -//convert a string into arango-key safe hex bytes hash -func hashString(dir string) string { - h := md5.New() - io.WriteString(h, dir) - b := h.Sum(nil) - return hex.EncodeToString(b) -} - -func bytesToArray(bs []byte) []uint64 { - out := make([]uint64, 0, 2+len(bs)/8) - out = append(out, uint64(len(bs))) - for len(bs)%8 != 0 { - bs = append(bs, 0) - } - for i := 0; i < len(bs); i = i + 8 { - out = append(out, binary.BigEndian.Uint64(bs[i:])) - } - return out -} - -func arrayToBytes(xs []uint64) []byte { - if len(xs) < 2 { - return []byte{} - } - first := xs[0] - out := make([]byte, len(xs)*8) - for i := 1; i < len(xs); i = i + 1 { - binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i]) - } - return out[:first] -} diff --git a/weed/filer/arangodb/arangodb_store_bucket.go b/weed/filer/arangodb/arangodb_store_bucket.go new file mode 100644 index 000000000..907328bda --- /dev/null +++ b/weed/filer/arangodb/arangodb_store_bucket.go @@ -0,0 +1,30 @@ +package arangodb + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "time" + + "github.com/chrislusf/seaweedfs/weed/glog" +) + +var _ filer.BucketAware = (*ArangodbStore)(nil) + +func (store *ArangodbStore) OnBucketCreation(bucket string) { + //nothing needs to be done +} +func (store *ArangodbStore) OnBucketDeletion(bucket string) { + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cur, err := store.database.Query(timeout, ` +for d in files +filter d.bucket == @bucket +remove d in files`, map[string]interface{}{"bucket": bucket}) + if err != nil { + glog.V(0).Infof("bucket delete %s : %v", bucket, err) + } + defer cur.Close() +} +func (store *ArangodbStore) CanDropWholeBucket() bool { + return true +} diff --git a/weed/filer/arangodb/arangodb_store_kv.go b/weed/filer/arangodb/arangodb_store_kv.go index 93caa75ed..2978f3dbe 100644 --- a/weed/filer/arangodb/arangodb_store_kv.go +++ b/weed/filer/arangodb/arangodb_store_kv.go @@ -4,16 +4,15 @@ import ( "context" "fmt" + "github.com/arangodb/go-driver" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" ) func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - dir, name := genDirAndName(key) model := &Model{ - Key: hashString(string(key)), - Directory: dir, - Name: name, + Key: hashString(".kvstore." + string(key)), + Directory: ".kvstore." + string(key), Meta: bytesToArray(value), } @@ -32,31 +31,24 @@ func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) return nil } - func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { var model Model - _, err = store.collection.ReadDocument(ctx, hashString(string(key)), &model) + _, err = store.collection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model) + if driver.IsNotFound(err) { + return nil, filer.ErrKvNotFound + } if err != nil { - glog.Errorf("kv get: %v", err) + glog.Errorf("kv get: %s %v", string(key), err) return nil, filer.ErrKvNotFound } return arrayToBytes(model.Meta), nil } func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.collection.RemoveDocument(ctx, hashString(string(key))) + _, err = store.collection.RemoveDocument(ctx, hashString(".kvstore."+string(key))) if err != nil { glog.Errorf("kv del: %v", err) return filer.ErrKvNotFound } return nil } - -func genDirAndName(key []byte) (dir string, name string) { - for len(key) < 8 { - key = append(key, 0) - } - dir = string(key[:8]) - name = string(key[8:]) - return -} diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go new file mode 100644 index 000000000..cf59957a6 --- /dev/null +++ b/weed/filer/arangodb/helpers.go @@ -0,0 +1,44 @@ +package arangodb + +import ( + "crypto/md5" + "encoding/binary" + "encoding/hex" + "io" +) + +//convert a string into arango-key safe hex bytes hash +func hashString(dir string) string { + h := md5.New() + io.WriteString(h, dir) + b := h.Sum(nil) + return hex.EncodeToString(b) +} + +// convert slice of bytes into slice of uint64 +// the first uint64 indicates the length in bytes +func bytesToArray(bs []byte) []uint64 { + out := make([]uint64, 0, 2+len(bs)/8) + out = append(out, uint64(len(bs))) + for len(bs)%8 != 0 { + bs = append(bs, 0) + } + for i := 0; i < len(bs); i = i + 8 { + out = append(out, binary.BigEndian.Uint64(bs[i:])) + } + return out +} + +// convert from slice of uint64 back to bytes +// if input length is 0 or 1, will return nil +func arrayToBytes(xs []uint64) []byte { + if len(xs) < 2 { + return nil + } + first := xs[0] + out := make([]byte, len(xs)*8) // i think this can actually be len(xs)*8-8, but i dont think an extra 8 bytes hurts... + for i := 1; i < len(xs); i = i + 1 { + binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i]) + } + return out[:first] +} diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md new file mode 100644 index 000000000..6cd187305 --- /dev/null +++ b/weed/filer/arangodb/readme.md @@ -0,0 +1,29 @@ +##arangodb + +database: https://github.com/arangodb/arangodb +go driver: https://github.com/arangodb/go-driver + + +options: + +``` +[arangodb] +enabled=true +db_name="seaweedfs" +servers=["http://localhost:8529"] +user="root" +pass="test" + +# whether to enable fulltext index +# this allows for directory prefix query +fulltext=true + +# tls settings +insecure_skip_verify=true +``` + +supports buckets with an extra field in document. +omitempty means extra space is not used. + +i test with +`docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0` diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index c774f5d27..27e68433d 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -25,9 +25,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR if findErr != nil { return findErr } - isDeleteCollection := f.isBucket(entry) - if entry.IsDirectory() { // delete the folder children, not including the folder itself err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(chunks []*filer_pb.FileChunk) error { From 2f0cdcdceb945be7490ec41ba857abcba133e892 Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 21:26:16 -0500 Subject: [PATCH 10/50] update readme --- weed/filer/arangodb/readme.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index 6cd187305..24f3a1e7e 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -27,3 +27,22 @@ omitempty means extra space is not used. i test with `docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0` + + +## todo + +implement native TTL + + +## thoughts + +should there be one collection per bucket? would make deleting a bucket instant as compared to fast + + +## comparison + +arangodb uses rocksdb in the background, so i am assuming things run in log time + +single document retreval might run in constant time + +i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion From 171c27ddf1a3889304ae36892a53c43076b5a4cb Mon Sep 17 00:00:00 2001 From: elee Date: Thu, 17 Mar 2022 22:35:30 -0500 Subject: [PATCH 11/50] zz --- weed/filer/arangodb/arangodb_store.go | 16 ++++++++++++++++ weed/filer/arangodb/readme.md | 3 +-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index a9f55f5bc..a39735108 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -33,6 +33,7 @@ type Model struct { Directory string `json:"directory,omitempty"` Name string `json:"name,omitempty"` Bucket string `json:"bucket,omitempty"` + Ttl string `json:"ttl,omitempty"` //arangodb does not support binary blobs //we encode byte slice into uint64 slice @@ -108,10 +109,15 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string, }); err != nil { return err } + if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } + if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1, + &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { + return err + } if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ Name: "IDX_name", @@ -188,6 +194,11 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) Meta: bytesToArray(meta), Bucket: bucket, } + if entry.TtlSec > 0 { + model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339) + } else { + model.Ttl = "" + } _, err = store.collection.CreateDocument(ctx, model) if driver.IsConflict(err) { return store.UpdateEntry(ctx, entry) @@ -231,6 +242,11 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) Name: name, Meta: bytesToArray(meta), } + if entry.TtlSec > 0 { + model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339) + } else { + model.Ttl = "none" + } _, err = store.collection.UpdateDocument(ctx, model.Key, model) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index 24f3a1e7e..eb6411c2e 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -43,6 +43,5 @@ should there be one collection per bucket? would make deleting a bucket instant arangodb uses rocksdb in the background, so i am assuming things run in log time -single document retreval might run in constant time - i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion +might need to change that From 0701feeb1791abefaad47e585170950922cf792d Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:26:11 -0500 Subject: [PATCH 12/50] added more to readme --- weed/filer/arangodb/arangodb_store.go | 6 +++--- weed/filer/arangodb/readme.md | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index a39735108..c1445f026 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -109,11 +109,11 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string, }); err != nil { return err } - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { return err } + if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1, &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { return err @@ -291,8 +291,8 @@ func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullP } func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { - query := "" - query = fmt.Sprintf(`for d in files filter starts_with(d.directory, "%s") remove d._key in files`, + var query string + query = query + fmt.Sprintf(`filter starts_with(d.directory, "%s") remove d._key in files`, strings.Join(strings.Split(string(fullpath), "/"), ","), string(fullpath), ) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index eb6411c2e..eaa8bd7a7 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -44,4 +44,9 @@ should there be one collection per bucket? would make deleting a bucket instant arangodb uses rocksdb in the background, so i am assuming things run in log time i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion -might need to change that +might need to change that. + + +ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct +it should be log time to the number of files in the directory +and constant time if you have full directory + file From 06f23aa6759155a6d5b713d8704a90e2e2d246d8 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:29:48 -0500 Subject: [PATCH 13/50] put in delete folder children query --- weed/filer/arangodb/arangodb_store.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index c1445f026..23aefe40f 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -292,7 +292,10 @@ func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullP func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { var query string - query = query + fmt.Sprintf(`filter starts_with(d.directory, "%s") remove d._key in files`, + query = query + fmt.Sprintf(` + for d in files + filter starts_with(d.directory, "%s/") || d.directory == "%s" + remove d._key in files`, strings.Join(strings.Split(string(fullpath), "/"), ","), string(fullpath), ) From 25be96832a910394a89e3c70f87edc4a34379a3e Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:33:08 -0500 Subject: [PATCH 14/50] update readme with index info --- weed/filer/arangodb/readme.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index eaa8bd7a7..966fec041 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -41,12 +41,10 @@ should there be one collection per bucket? would make deleting a bucket instant ## comparison -arangodb uses rocksdb in the background, so i am assuming things run in log time - -i am not sure how the prefix query scales compared to the recursive calls that some other stores do for folder deletion -might need to change that. - - ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct + it should be log time to the number of files in the directory + and constant time if you have full directory + file + +deleting a folder should be log time to number of folders + files that need to be deleted From 1cea6c73d30ddee95b42335af74701a61c19ec84 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 00:34:19 -0500 Subject: [PATCH 15/50] update readme --- weed/filer/arangodb/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index 966fec041..e56012d8c 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -31,12 +31,12 @@ i test with ## todo -implement native TTL +performance test ## thoughts -should there be one collection per bucket? would make deleting a bucket instant as compared to fast +should there be one collection per bucket? this would make deleting a bucket O(1) instead of O(n) ## comparison From 411c0df3fec3344c3e5538cf56b54c1567162386 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 21:51:16 -0500 Subject: [PATCH 16/50] switch to multi collection, change readme --- weed/filer/arangodb/arangodb_store.go | 215 +++++++------------ weed/filer/arangodb/arangodb_store_bucket.go | 26 ++- weed/filer/arangodb/arangodb_store_kv.go | 10 +- weed/filer/arangodb/helpers.go | 92 ++++++++ weed/filer/arangodb/readme.md | 44 ++-- 5 files changed, 217 insertions(+), 170 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 23aefe40f..d27799b0e 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -4,7 +4,9 @@ import ( "context" "crypto/tls" "fmt" + "strconv" "strings" + "sync" "time" "github.com/arangodb/go-driver" @@ -19,11 +21,20 @@ func init() { filer.Stores = append(filer.Stores, &ArangodbStore{}) } +var ( + BUCKET_PREFIX = "/buckets" + DEFAULT_COLLECTION = "seaweed_no_bucket" + KVMETA_COLLECTION = "seaweed_kvmeta" +) + type ArangodbStore struct { - connect driver.Connection - client driver.Client - database driver.Database - collection driver.Collection + connect driver.Connection + client driver.Client + database driver.Database + kvCollection driver.Collection + + buckets map[string]driver.Collection + mu sync.RWMutex databaseName string } @@ -32,7 +43,6 @@ type Model struct { Key string `json:"_key"` Directory string `json:"directory,omitempty"` Name string `json:"name,omitempty"` - Bucket string `json:"bucket,omitempty"` Ttl string `json:"ttl,omitempty"` //arangodb does not support binary blobs @@ -46,6 +56,7 @@ func (store *ArangodbStore) GetName() string { } func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) { + store.buckets = make(map[string]driver.Collection, 3) store.databaseName = configuration.GetString(prefix + "db_name") return store.connection(configuration.GetStringSlice(prefix+"servers"), configuration.GetString(prefix+"user"), @@ -85,49 +96,7 @@ func (store *ArangodbStore) connection(uris []string, user string, pass string, if err != nil { return err } - - coll_name := "files" - ok, err = store.database.CollectionExists(ctx, coll_name) - if err != nil { - return err - } - if ok { - store.collection, err = store.database.Collection(ctx, coll_name) - } else { - store.collection, err = store.database.CreateCollection(ctx, coll_name, &driver.CreateCollectionOptions{}) - } - if err != nil { - return err - } - - // ensure indices - - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory", "name"}, - &driver.EnsurePersistentIndexOptions{ - Name: "directory_name_multi", - Unique: true, - }); err != nil { - return err - } - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"directory"}, - &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { - return err - } - - if _, _, err = store.collection.EnsureTTLIndex(ctx, "ttl", 1, - &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { - return err - } - - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ - Name: "IDX_name", - }); err != nil { - return err - } - if _, _, err = store.collection.EnsurePersistentIndex(ctx, []string{"bucket"}, &driver.EnsurePersistentIndexOptions{ - Name: "IDX_bucket", - Sparse: true, //sparse index, to locate files of bucket - }); err != nil { + if store.kvCollection, err = store.ensureCollection(ctx, KVMETA_COLLECTION); err != nil { return err } return err @@ -140,8 +109,13 @@ const ( ) func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) { + keys := make([]string, 0, len(store.buckets)+1) + for k := range store.buckets { + keys = append(keys, k) + } + keys = append(keys, store.kvCollection.Name()) txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{ - Exclusive: []string{"files"}, + Exclusive: keys, }, &driver.BeginTransactionOptions{}) if err != nil { return nil, err @@ -186,23 +160,27 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) if len(entry.Chunks) > 50 { meta = util.MaybeGzipData(meta) } - bucket, _ := extractBucket(entry.FullPath) model := &Model{ Key: hashString(string(entry.FullPath)), Directory: dir, Name: name, Meta: bytesToArray(meta), - Bucket: bucket, } if entry.TtlSec > 0 { model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339) } else { model.Ttl = "" } - _, err = store.collection.CreateDocument(ctx, model) + + targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath) + if err != nil { + return err + } + _, err = targetCollection.CreateDocument(ctx, model) if driver.IsConflict(err) { return store.UpdateEntry(ctx, entry) } + if err != nil { return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err) } @@ -211,21 +189,6 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) } -func extractBucket(fullpath util.FullPath) (string, string) { - if !strings.HasPrefix(string(fullpath), "/buckets/") { - return "", string(fullpath) - } - bucketAndObjectKey := string(fullpath)[len("/buckets/"):] - t := strings.Index(bucketAndObjectKey, "/") - bucket := bucketAndObjectKey - shortPath := "/" - if t > 0 { - bucket = bucketAndObjectKey[:t] - shortPath = string(util.FullPath(bucketAndObjectKey[t:])) - } - return bucket, shortPath -} - func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -247,9 +210,11 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) } else { model.Ttl = "none" } - - _, err = store.collection.UpdateDocument(ctx, model.Key, model) - + targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath) + if err != nil { + return err + } + _, err = targetCollection.UpdateDocument(ctx, model.Key, model) if err != nil { return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err) } @@ -259,11 +224,15 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { var data Model - _, err = store.collection.ReadDocument(ctx, hashString(string(fullpath)), &data) - if driver.IsNotFound(err) { - return nil, filer_pb.ErrNotFound - } + targetCollection, err := store.extractBucketCollection(ctx, fullpath) if err != nil { + return nil, err + } + _, err = targetCollection.ReadDocument(ctx, hashString(string(fullpath)), &data) + if err != nil { + if driver.IsNotFound(err) { + return nil, filer_pb.ErrNotFound + } glog.Errorf("find %s: %v", fullpath, err) return nil, filer_pb.ErrNotFound } @@ -281,8 +250,12 @@ func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPat return entry, nil } -func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { - _, err := store.collection.RemoveDocument(ctx, hashString(string(fullpath))) +func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + targetCollection, err := store.extractBucketCollection(ctx, fullpath) + if err != nil { + return err + } + _, err = targetCollection.RemoveDocument(ctx, hashString(string(fullpath))) if err != nil && !driver.IsNotFound(err) { glog.Errorf("find %s: %v", fullpath, err) return fmt.Errorf("delete %s : %v", fullpath, err) @@ -290,14 +263,21 @@ func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullP return nil } -func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { +// this runs in log time +func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { var query string + targetCollection, err := store.extractBucketCollection(ctx, fullpath) + if err != nil { + return err + } query = query + fmt.Sprintf(` - for d in files + for d in %s filter starts_with(d.directory, "%s/") || d.directory == "%s" - remove d._key in files`, + remove d._key in %s`, + targetCollection.Name(), strings.Join(strings.Split(string(fullpath), "/"), ","), string(fullpath), + targetCollection.Name(), ) cur, err := store.database.Query(ctx, query, nil) if err != nil { @@ -307,26 +287,33 @@ func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath u return nil } +func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc) +} + func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - // if no prefix, then dont use index - if prefix == "" { - return store.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc) + targetCollection, err := store.extractBucketCollection(ctx, dirPath) + if err != nil { + return lastFileName, err } - eq := "" + query := "for d in " + targetCollection.Name() if includeStartFile { - eq = "filter d.name >= \"" + startFileName + "\"" + query = query + " filter d.name >= \"" + startFileName + "\" " } else { - eq = "filter d.name > \"" + startFileName + "\"" + query = query + " filter d.name > \"" + startFileName + "\" " } - query := fmt.Sprintf(` -for d in files + if prefix != "" { + query = query + fmt.Sprintf(`&& starts_with(d.name, "%s")`, prefix) + } + query = query + ` filter d.directory == @dir -filter starts_with(d.name, @prefix) -%s sort d.name asc -limit %d -return d`, eq, limit) - cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath, "prefix": prefix}) +` + if limit > 0 { + query = query + "limit " + strconv.Itoa(int(limit)) + } + query = query + "\n return d" + cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath}) if err != nil { return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) } @@ -356,49 +343,5 @@ return d`, eq, limit) return lastFileName, err } -func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - eq := "" - if includeStartFile { - eq = "filter d.name >= \"" + startFileName + "\"" - } else { - eq = "filter d.name > \"" + startFileName + "\"" - } - query := fmt.Sprintf(` -for d in files -filter d.directory == "%s" -%s -sort d.name asc -limit %d -return d`, string(dirPath), eq, limit) - cur, err := store.database.Query(ctx, query, nil) - if err != nil { - return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err) - } - defer cur.Close() - for cur.HasMore() { - var data Model - _, err = cur.ReadDocument(ctx, &data) - if err != nil { - break - } - entry := &filer.Entry{ - FullPath: util.NewFullPath(string(dirPath), data.Name), - } - lastFileName = data.Name - converted := arrayToBytes(data.Meta) - if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil { - err = decodeErr - glog.V(0).Infof("list %s : %v", entry.FullPath, err) - break - } - - if !eachEntryFunc(entry) { - break - } - - } - return lastFileName, err -} - func (store *ArangodbStore) Shutdown() { } diff --git a/weed/filer/arangodb/arangodb_store_bucket.go b/weed/filer/arangodb/arangodb_store_bucket.go index 907328bda..63d407309 100644 --- a/weed/filer/arangodb/arangodb_store_bucket.go +++ b/weed/filer/arangodb/arangodb_store_bucket.go @@ -2,28 +2,38 @@ package arangodb import ( "context" - "github.com/chrislusf/seaweedfs/weed/filer" "time" + "github.com/arangodb/go-driver" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" ) var _ filer.BucketAware = (*ArangodbStore)(nil) func (store *ArangodbStore) OnBucketCreation(bucket string) { - //nothing needs to be done + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // create the collection && add to cache + _, err := store.ensureBucket(timeout, bucket) + if err != nil { + glog.V(0).Infof("bucket create %s : %w", bucket, err) + } } func (store *ArangodbStore) OnBucketDeletion(bucket string) { timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - cur, err := store.database.Query(timeout, ` -for d in files -filter d.bucket == @bucket -remove d in files`, map[string]interface{}{"bucket": bucket}) + collection, err := store.ensureBucket(timeout, bucket) if err != nil { - glog.V(0).Infof("bucket delete %s : %v", bucket, err) + glog.V(0).Infof("bucket delete %s : %w", bucket, err) + return + } + err = collection.Remove(timeout) + if err != nil && !driver.IsNotFound(err) { + glog.V(0).Infof("bucket delete %s : %w", bucket, err) + return } - defer cur.Close() } func (store *ArangodbStore) CanDropWholeBucket() bool { return true diff --git a/weed/filer/arangodb/arangodb_store_kv.go b/weed/filer/arangodb/arangodb_store_kv.go index 2978f3dbe..c1307e78d 100644 --- a/weed/filer/arangodb/arangodb_store_kv.go +++ b/weed/filer/arangodb/arangodb_store_kv.go @@ -16,14 +16,14 @@ func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) Meta: bytesToArray(value), } - exists, err := store.collection.DocumentExists(ctx, model.Key) + exists, err := store.kvCollection.DocumentExists(ctx, model.Key) if err != nil { return fmt.Errorf("kv put: %v", err) } if exists { - _, err = store.collection.UpdateDocument(ctx, model.Key, model) + _, err = store.kvCollection.UpdateDocument(ctx, model.Key, model) } else { - _, err = store.collection.CreateDocument(ctx, model) + _, err = store.kvCollection.CreateDocument(ctx, model) } if err != nil { return fmt.Errorf("kv put: %v", err) @@ -33,7 +33,7 @@ func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) } func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { var model Model - _, err = store.collection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model) + _, err = store.kvCollection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model) if driver.IsNotFound(err) { return nil, filer.ErrKvNotFound } @@ -45,7 +45,7 @@ func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte } func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) { - _, err = store.collection.RemoveDocument(ctx, hashString(".kvstore."+string(key))) + _, err = store.kvCollection.RemoveDocument(ctx, hashString(".kvstore."+string(key))) if err != nil { glog.Errorf("kv del: %v", err) return filer.ErrKvNotFound diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go index cf59957a6..c91ef2be5 100644 --- a/weed/filer/arangodb/helpers.go +++ b/weed/filer/arangodb/helpers.go @@ -1,10 +1,15 @@ package arangodb import ( + "context" "crypto/md5" "encoding/binary" "encoding/hex" "io" + "strings" + + "github.com/arangodb/go-driver" + "github.com/chrislusf/seaweedfs/weed/util" ) //convert a string into arango-key safe hex bytes hash @@ -42,3 +47,90 @@ func arrayToBytes(xs []uint64) []byte { } return out[:first] } + +// gets the bucket name out of filepath +func extractBucket(fullpath util.FullPath) (string, string) { + if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") { + return "", string(fullpath) + } + if strings.Count(string(fullpath), "/") < 2 { + return "", string(fullpath) + } + bucketAndObjectKey := string(fullpath)[len("/buckets/"):] + t := strings.Index(bucketAndObjectKey, "/") + bucket := bucketAndObjectKey + shortPath := "/" + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = string(util.FullPath(bucketAndObjectKey[t:])) + } + return bucket, shortPath +} + +// gets the collection the bucket points to from filepath +func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) { + bucket, _ := extractBucket(fullpath) + if bucket == "" { + bucket = DEFAULT_COLLECTION + } + c, err = store.ensureBucket(ctx, bucket) + if err != nil { + return nil, err + } + return c, err +} + +// get bucket collection from cache. if not exist, creates the buckets collection and grab it +func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) { + var ok bool + store.mu.RLock() + bc, ok = store.buckets[bucket] + store.mu.RUnlock() + if ok { + return bc, nil + } + store.mu.Lock() + defer store.mu.Unlock() + store.buckets[bucket], err = store.ensureCollection(ctx, bucket) + if err != nil { + return nil, err + } + return store.buckets[bucket], nil +} + +// creates collection if not exist, ensures indices if not exist +func (store *ArangodbStore) ensureCollection(ctx context.Context, name string) (c driver.Collection, err error) { + ok, err := store.database.CollectionExists(ctx, name) + if err != nil { + return + } + if ok { + c, err = store.database.Collection(ctx, name) + } else { + c, err = store.database.CreateCollection(ctx, name, &driver.CreateCollectionOptions{}) + } + if err != nil { + return + } + // ensure indices + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory", "name"}, + &driver.EnsurePersistentIndexOptions{ + Name: "directory_name_multi", Unique: true, + }); err != nil { + return + } + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory"}, + &driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil { + return + } + if _, _, err = c.EnsureTTLIndex(ctx, "ttl", 1, + &driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil { + return + } + if _, _, err = c.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{ + Name: "IDX_name", + }); err != nil { + return + } + return c, nil +} diff --git a/weed/filer/arangodb/readme.md b/weed/filer/arangodb/readme.md index e56012d8c..e189811fb 100644 --- a/weed/filer/arangodb/readme.md +++ b/weed/filer/arangodb/readme.md @@ -3,7 +3,6 @@ database: https://github.com/arangodb/arangodb go driver: https://github.com/arangodb/go-driver - options: ``` @@ -11,40 +10,43 @@ options: enabled=true db_name="seaweedfs" servers=["http://localhost:8529"] +#basic auth user="root" pass="test" -# whether to enable fulltext index -# this allows for directory prefix query -fulltext=true - # tls settings insecure_skip_verify=true ``` -supports buckets with an extra field in document. -omitempty means extra space is not used. - -i test with +i test using this dev database: `docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0` -## todo - -performance test +## features i don't personally need but are missing + [ ] provide tls cert to arango + [ ] authentication that is not basic auth + [ ] synchronise endpoint interval config + [ ] automatic creation of custom index + [ ] configure default arangodb collection sharding rules + [ ] configure default arangodb collection replication rules -## thoughts - -should there be one collection per bucket? this would make deleting a bucket O(1) instead of O(n) - - -## comparison +## complexity ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct -it should be log time to the number of files in the directory +O(1) +- InsertEntry +- UpdateEntry +- FindEntry +- DeleteEntry +- KvPut +- KvGet +- KvDelete -and constant time if you have full directory + file +O(log(BUCKET_SIZE)) +- DeleteFolderChildren -deleting a folder should be log time to number of folders + files that need to be deleted +O(log(DIRECTORY_SIZE)) +- ListDirectoryEntries +- ListDirectoryPrefixedEntries From beb406bbbb5e67fc66a02d264c670a8fe11785b7 Mon Sep 17 00:00:00 2001 From: elee Date: Fri, 18 Mar 2022 22:21:57 -0500 Subject: [PATCH 17/50] fix ls, onBucketDelete still not triggering --- weed/filer/arangodb/arangodb_store.go | 2 +- weed/filer/arangodb/helpers.go | 38 +++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index d27799b0e..27ed9132b 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -292,7 +292,7 @@ func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath ut } func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - targetCollection, err := store.extractBucketCollection(ctx, dirPath) + targetCollection, err := store.extractBucketCollection(ctx, dirPath+"/") if err != nil { return lastFileName, err } diff --git a/weed/filer/arangodb/helpers.go b/weed/filer/arangodb/helpers.go index c91ef2be5..943189781 100644 --- a/weed/filer/arangodb/helpers.go +++ b/weed/filer/arangodb/helpers.go @@ -48,25 +48,6 @@ func arrayToBytes(xs []uint64) []byte { return out[:first] } -// gets the bucket name out of filepath -func extractBucket(fullpath util.FullPath) (string, string) { - if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") { - return "", string(fullpath) - } - if strings.Count(string(fullpath), "/") < 2 { - return "", string(fullpath) - } - bucketAndObjectKey := string(fullpath)[len("/buckets/"):] - t := strings.Index(bucketAndObjectKey, "/") - bucket := bucketAndObjectKey - shortPath := "/" - if t > 0 { - bucket = bucketAndObjectKey[:t] - shortPath = string(util.FullPath(bucketAndObjectKey[t:])) - } - return bucket, shortPath -} - // gets the collection the bucket points to from filepath func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) { bucket, _ := extractBucket(fullpath) @@ -80,6 +61,25 @@ func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpat return c, err } +// called by extractBucketCollection +func extractBucket(fullpath util.FullPath) (string, string) { + if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") { + return "", string(fullpath) + } + if strings.Count(string(fullpath), "/") < 3 { + return "", string(fullpath) + } + bucketAndObjectKey := string(fullpath)[len(BUCKET_PREFIX+"/"):] + t := strings.Index(bucketAndObjectKey, "/") + bucket := bucketAndObjectKey + shortPath := "/" + if t > 0 { + bucket = bucketAndObjectKey[:t] + shortPath = string(util.FullPath(bucketAndObjectKey[t:])) + } + return bucket, shortPath +} + // get bucket collection from cache. if not exist, creates the buckets collection and grab it func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) { var ok bool From 6701ac3a61a052a1f29b4a3fe852d53d7375845a Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:14:49 +0000 Subject: [PATCH 18/50] Merge branch 'chrislusf-master' into a --- go.mod | 2 +- go.sum | 20 +------------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 06e686e17..149b4894a 100644 --- a/go.mod +++ b/go.mod @@ -155,6 +155,7 @@ require ( ) require ( + github.com/arangodb/go-driver v1.2.1 github.com/fluent/fluent-logger-golang v1.8.0 github.com/hanwen/go-fuse/v2 v2.1.0 ) @@ -163,7 +164,6 @@ require ( cloud.google.com/go/compute v1.5.0 // indirect cloud.google.com/go/iam v0.1.1 // indirect github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect - github.com/arangodb/go-driver v1.2.1 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect diff --git a/go.sum b/go.sum index 15350f903..becd05248 100644 --- a/go.sum +++ b/go.sum @@ -178,15 +178,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -<<<<<<< HEAD -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8= github.com/arangodb/go-driver v1.2.1/go.mod h1:zdDkJJnCj8DAkfbtIjIXnsTrWIiy6VhP3Vy14p+uQeY= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= -======= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -283,11 +278,8 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -<<<<<<< HEAD github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= -======= github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -393,18 +385,11 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -<<<<<<< HEAD -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -======= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -793,14 +778,10 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -<<<<<<< HEAD github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -======= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= ->>>>>>> 93615b2a49b6510e6108a784b54be7d16719d730 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seaweedfs/goexif v2.0.0+incompatible h1:x8pckiT12QQhifwhDQpeISgDfsqmQ6VR4LFPQ64JRps= @@ -1229,6 +1210,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 71307ce2e6ee06a39cf9c378ad27a548cb950e1b Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:25:53 +0000 Subject: [PATCH 19/50] test ci --- .gitlab-ci.yml | 25 ++ .vscode/configurationCache.log | 1 + .vscode/dryrun.log | 6 + .vscode/settings.json | 3 + .vscode/targets.log | 630 +++++++++++++++++++++++++++++++++ 5 files changed, 665 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 .vscode/configurationCache.log create mode 100644 .vscode/dryrun.log create mode 100644 .vscode/settings.json create mode 100644 .vscode/targets.log diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..9a17a8d7d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,25 @@ + +build_binary_latest: + stage: build + image: + name: golang:1.18.0-alpine3.15 + script: + - go build -o ./weed/weed ./weed + rules: + - if: $CI_COMMIT_TAG + +build_container_latest: + stage: build + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + - >- + /kaniko/executor + --context "${CI_PROJECT_DIR}/docker" + --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" + rules: + - if: $CI_COMMIT_TAG \ No newline at end of file diff --git a/.vscode/configurationCache.log b/.vscode/configurationCache.log new file mode 100644 index 000000000..bab9054b3 --- /dev/null +++ b/.vscode/configurationCache.log @@ -0,0 +1 @@ +{"buildTargets":[],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}} \ No newline at end of file diff --git a/.vscode/dryrun.log b/.vscode/dryrun.log new file mode 100644 index 000000000..40a61ed1e --- /dev/null +++ b/.vscode/dryrun.log @@ -0,0 +1,6 @@ +make --dry-run --always-make --keep-going --print-directory +make: Entering directory '/home/coder/repos/seaweedfs' +make: Leaving directory '/home/coder/repos/seaweedfs' + +make: *** No targets specified and no makefile found. Stop. + diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..65e1ec078 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "makefile.extensionOutputFolder": "./.vscode" +} \ No newline at end of file diff --git a/.vscode/targets.log b/.vscode/targets.log new file mode 100644 index 000000000..6684e3315 --- /dev/null +++ b/.vscode/targets.log @@ -0,0 +1,630 @@ +make all --print-data-base --no-builtin-variables --no-builtin-rules --question +make: *** No rule to make target 'all'. Stop. + +# GNU Make 4.3 +# Built for x86_64-pc-linux-gnu +# Copyright (C) 1988-2020 Free Software Foundation, Inc. +# License GPLv3+: GNU GPL version 3 or later +# This is free software: you are free to change and redistribute it. +# There is NO WARRANTY, to the extent permitted by law. + +# Make data base, printed on Mon Mar 28 17:22:08 2022 + +# Variables + +# environment +SEAWEED_VOLUME_1_SERVICE_PORT = 8444 +# environment +GETTY_CODE_CODE_SERVER_SERVICE_PORT = 8080 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PROTO = tcp +# environment +KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_CR_METRICS = 8686 +# environment +PLIK_SERVICE_HOST = 10.245.176.243 +# environment +SEAWEED_VOLUME_0_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +LC_ALL = C +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP = tcp://10.245.169.206:8686 +# environment +CODE_SERVER_SERVICE_HOST = 10.245.153.237 +# environment +PG_HA_HA_SERVICE_PORT = 5432 +# environment +SEAWEED_FILER_PORT_8333_TCP_PROTO = tcp +# environment +REDIS_REPLICAS_SERVICE_PORT_TCP_REDIS = 6379 +# environment +SEAWEED_FILER_PORT_8888_TCP = tcp://10.245.202.58:8888 +# environment +VSCODE_INJECT_NODE_MODULE_LOOKUP_PATH = /usr/lib/code-server/vendor/modules/code-oss-dev/remote/node_modules +# environment +REDIS_MASTER_SERVICE_HOST = 10.245.29.137 +# environment +REDIS_MASTER_PORT_6379_TCP = tcp://10.245.29.137:6379 +# environment +SEAWEED_FILER_PORT = tcp://10.245.202.58:8888 +# environment +REDIS_REPLICAS_PORT_6379_TCP = tcp://10.245.122.67:6379 +# environment +VSCODE_CWD = /home/coder +# environment +KUBERNETES_PORT_443_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_3_PORT_18444_TCP_ADDR = 10.245.205.95 +# environment +TRAEFIK_PORT_443_TCP = tcp://10.245.52.194:443 +# environment +SEAWEED_FILER_SERVICE_PORT_FILER_HTTP = 8888 +# environment +SEAWEED_VOLUME_2_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +PG_HA_REPLICAS_PORT_5432_TCP_PORT = 5432 +# environment +TRAEFIK_PORT_443_TCP_ADDR = 10.245.52.194 +# environment +CODE_SERVER_PARENT_PID = 12 +# default +MAKE_COMMAND := make +# environment +SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +SEAWEED_VOLUME_3_PORT_8444_TCP_PORT = 8444 +# environment +PASTEY_PORT_5000_TCP_PORT = 5000 +# environment +KUBERNETES_PORT_443_TCP_PORT = 443 +# environment +PG_HA_PGBOUNCER_PORT_5432_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_GRPC = 18444 +# environment +GOPATH = /home/coder/.asdf/installs/golang/1.18/packages +# automatic +@D = $(patsubst %/,%,$(dir $@)) +# environment +PG_HA_REPLICAS_SERVICE_PORT = 5432 +# environment +PG_HA_HA_PORT_5432_TCP_PORT = 5432 +# environment +GETTY_CODE_CODE_SERVER_SERVICE_HOST = 10.245.62.90 +# environment +PGADMIN4_SERVICE_HOST = 10.245.182.193 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_0_PORT = tcp://10.245.135.162:8444 +# environment +SEAWEED_FILER_PORT_8888_TCP_PROTO = tcp +# environment +SEAWEED_MASTER_PORT_19333_TCP = tcp://10.245.145.57:19333 +# environment +REDIS_MASTER_SERVICE_PORT = 6379 +# environment +SEAWEED_VOLUME_0_PORT_18444_TCP_PORT = 18444 +# environment +VSCODE_HANDLES_UNCAUGHT_ERRORS = true +# default +.VARIABLES := +# environment +PWD = /home/coder/repos/seaweedfs +# automatic +%D = $(patsubst %/,%,$(dir $%)) +# environment +HOSTNAME = code-server-5dfcd6ff77-w8kdz +# environment +PG_HA_PGBOUNCER_PORT_5432_TCP = tcp://10.245.186.4:5432 +# environment +SEAWEED_VOLUME_2_PORT_8444_TCP = tcp://10.245.184.0:8444 +# environment +PG_HA_HA_SERVICE_PORT_POSTGRES = 5432 +# environment +TRAEFIK_PORT_80_TCP_ADDR = 10.245.52.194 +# environment +PG_HA_REPLICAS_PORT = tcp://10.245.76.221:5432 +# environment +SEAWEED_FILER_PORT_18888_TCP = tcp://10.245.202.58:18888 +# automatic +^D = $(patsubst %/,%,$(dir $^)) +# environment +VSCODE_LOG_STACK = false +# automatic +%F = $(notdir $%) +# environment +PASTEY_SERVICE_PORT = 5000 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PORT = 8686 +# environment +LANG = C +# default +.LOADED := +# default +.INCLUDE_DIRS = /usr/local/include /usr/include /usr/include +# environment +PG_HA_HA_PORT_5432_TCP = tcp://10.245.23.155:5432 +# environment +TRAEFIK_PORT_22_TCP = tcp://10.245.52.194:22 +# makefile +MAKEFLAGS = pqrR +# environment +SEAWEED_VOLUME_1_SERVICE_PORT_VOLUME_HTTP = 8444 +# environment +REDIS_REPLICAS_PORT_6379_TCP_PROTO = tcp +# environment +TRAEFIK_SERVICE_PORT_WEBSECURE = 443 +# environment +REDIS_MASTER_SERVICE_PORT_TCP_REDIS = 6379 +# environment +SEAWEED_VOLUME_1_PORT_18444_TCP_PORT = 18444 +# environment +SEAWEED_VOLUME_2_SERVICE_PORT = 8444 +# makefile +CURDIR := /home/coder/repos/seaweedfs +# environment +SEAWEED_FILER_SERVICE_HOST = 10.245.202.58 +# environment +VSCODE_PIPE_LOGGING = true +# environment +SEAWEED_FILER_SERVICE_PORT_FILER_GRPC = 18888 +# environment +REDIS_REPLICAS_SERVICE_PORT = 6379 +# environment +PG_HA_PGBOUNCER_SERVICE_HOST = 10.245.186.4 +# environment +APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = true +# environment +PG_HA_REPLICAS_SERVICE_HOST = 10.245.76.221 +# automatic +*D = $(patsubst %/,%,$(dir $*)) +# environment +PGADMIN4_PORT_80_TCP_ADDR = 10.245.182.193 +# environment +SEAWEED_MASTER_PORT_9333_TCP = tcp://10.245.145.57:9333 +# environment +TRAEFIK_SERVICE_HOST = 10.245.52.194 +# default +.SHELLFLAGS := -c +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PORT = 8383 +# environment +KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_ADDR = 10.245.169.206 +# environment +MFLAGS = -pqrR +# automatic ++D = $(patsubst %/,%,$(dir $+)) +# environment +PASTEY_PORT_5000_TCP_PROTO = tcp +# makefile +MAKEFILE_LIST := +# environment +PG_HA_PGBOUNCER_SERVICE_PORT = 5432 +# environment +REDIS_MASTER_PORT_6379_TCP_ADDR = 10.245.29.137 +# environment +REDIS_REPLICAS_PORT = tcp://10.245.122.67:6379 +# environment +GETTY_CODE_CODE_SERVER_PORT_8080_TCP = tcp://10.245.62.90:8080 +# environment +PG_HA_REPLICAS_PORT_5432_TCP = tcp://10.245.76.221:5432 +# environment +SEAWEED_MASTER_PORT_9333_TCP_PROTO = tcp +# automatic +@F = $(notdir $@) +# environment +VSCODE_VERBOSE_LOGGING = true +# environment +TRAEFIK_PORT_443_TCP_PROTO = tcp +# automatic +?D = $(patsubst %/,%,$(dir $?)) +# environment +PASTEY_PORT = tcp://10.245.184.85:5000 +# environment +TRAEFIK_PORT_80_TCP_PROTO = tcp +# environment +SEAWEED_MASTER_PORT_9333_TCP_PORT = 9333 +# environment +GITEA_MEMCACHED_PORT = tcp://10.245.55.95:11211 +# environment +SEAWEED_VOLUME_0_PORT_18444_TCP_PROTO = tcp +# environment +SEAWEED_MASTER_PORT_19333_TCP_PROTO = tcp +# environment +SEAWEED_VOLUME_0_SERVICE_PORT = 8444 +# environment +PLIK_PORT_8080_TCP = tcp://10.245.176.243:8080 +# environment +KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_HTTP_METRICS = 8383 +# environment +SEAWEED_VOLUME_2_PORT_8444_TCP_ADDR = 10.245.184.0 +# environment +SEAWEED_VOLUME_2_PORT = tcp://10.245.184.0:8444 +# automatic +*F = $(notdir $*) +# environment +TRAEFIK_PORT_22_TCP_ADDR = 10.245.52.194 +# environment +KUBERNETES_PORT_443_TCP_ADDR = 10.245.0.1 +# environment +CODE_SERVER_PORT_8080_TCP = tcp://10.245.153.237:8080 +# environment +TRAEFIK_PORT_22_TCP_PORT = 22 +# automatic + Date: Mon, 28 Mar 2022 17:26:43 +0000 Subject: [PATCH 20/50] del --- .gitignore | 2 + .vscode/configurationCache.log | 1 - .vscode/dryrun.log | 6 - .vscode/settings.json | 3 - .vscode/targets.log | 630 --------------------------------- 5 files changed, 2 insertions(+), 640 deletions(-) delete mode 100644 .vscode/configurationCache.log delete mode 100644 .vscode/dryrun.log delete mode 100644 .vscode/settings.json delete mode 100644 .vscode/targets.log diff --git a/.gitignore b/.gitignore index e7ae92784..25a58bc67 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,8 @@ Temporary Items # Mongo Explorer plugin: # .idea/mongoSettings.xml +## vscode +.vscode ## File-based project format: *.ipr *.iws diff --git a/.vscode/configurationCache.log b/.vscode/configurationCache.log deleted file mode 100644 index bab9054b3..000000000 --- a/.vscode/configurationCache.log +++ /dev/null @@ -1 +0,0 @@ -{"buildTargets":[],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}} \ No newline at end of file diff --git a/.vscode/dryrun.log b/.vscode/dryrun.log deleted file mode 100644 index 40a61ed1e..000000000 --- a/.vscode/dryrun.log +++ /dev/null @@ -1,6 +0,0 @@ -make --dry-run --always-make --keep-going --print-directory -make: Entering directory '/home/coder/repos/seaweedfs' -make: Leaving directory '/home/coder/repos/seaweedfs' - -make: *** No targets specified and no makefile found. Stop. - diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 65e1ec078..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "makefile.extensionOutputFolder": "./.vscode" -} \ No newline at end of file diff --git a/.vscode/targets.log b/.vscode/targets.log deleted file mode 100644 index 6684e3315..000000000 --- a/.vscode/targets.log +++ /dev/null @@ -1,630 +0,0 @@ -make all --print-data-base --no-builtin-variables --no-builtin-rules --question -make: *** No rule to make target 'all'. Stop. - -# GNU Make 4.3 -# Built for x86_64-pc-linux-gnu -# Copyright (C) 1988-2020 Free Software Foundation, Inc. -# License GPLv3+: GNU GPL version 3 or later -# This is free software: you are free to change and redistribute it. -# There is NO WARRANTY, to the extent permitted by law. - -# Make data base, printed on Mon Mar 28 17:22:08 2022 - -# Variables - -# environment -SEAWEED_VOLUME_1_SERVICE_PORT = 8444 -# environment -GETTY_CODE_CODE_SERVER_SERVICE_PORT = 8080 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PROTO = tcp -# environment -KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_CR_METRICS = 8686 -# environment -PLIK_SERVICE_HOST = 10.245.176.243 -# environment -SEAWEED_VOLUME_0_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -LC_ALL = C -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP = tcp://10.245.169.206:8686 -# environment -CODE_SERVER_SERVICE_HOST = 10.245.153.237 -# environment -PG_HA_HA_SERVICE_PORT = 5432 -# environment -SEAWEED_FILER_PORT_8333_TCP_PROTO = tcp -# environment -REDIS_REPLICAS_SERVICE_PORT_TCP_REDIS = 6379 -# environment -SEAWEED_FILER_PORT_8888_TCP = tcp://10.245.202.58:8888 -# environment -VSCODE_INJECT_NODE_MODULE_LOOKUP_PATH = /usr/lib/code-server/vendor/modules/code-oss-dev/remote/node_modules -# environment -REDIS_MASTER_SERVICE_HOST = 10.245.29.137 -# environment -REDIS_MASTER_PORT_6379_TCP = tcp://10.245.29.137:6379 -# environment -SEAWEED_FILER_PORT = tcp://10.245.202.58:8888 -# environment -REDIS_REPLICAS_PORT_6379_TCP = tcp://10.245.122.67:6379 -# environment -VSCODE_CWD = /home/coder -# environment -KUBERNETES_PORT_443_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_3_PORT_18444_TCP_ADDR = 10.245.205.95 -# environment -TRAEFIK_PORT_443_TCP = tcp://10.245.52.194:443 -# environment -SEAWEED_FILER_SERVICE_PORT_FILER_HTTP = 8888 -# environment -SEAWEED_VOLUME_2_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -PG_HA_REPLICAS_PORT_5432_TCP_PORT = 5432 -# environment -TRAEFIK_PORT_443_TCP_ADDR = 10.245.52.194 -# environment -CODE_SERVER_PARENT_PID = 12 -# default -MAKE_COMMAND := make -# environment -SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -SEAWEED_VOLUME_3_PORT_8444_TCP_PORT = 8444 -# environment -PASTEY_PORT_5000_TCP_PORT = 5000 -# environment -KUBERNETES_PORT_443_TCP_PORT = 443 -# environment -PG_HA_PGBOUNCER_PORT_5432_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_3_SERVICE_PORT_VOLUME_GRPC = 18444 -# environment -GOPATH = /home/coder/.asdf/installs/golang/1.18/packages -# automatic -@D = $(patsubst %/,%,$(dir $@)) -# environment -PG_HA_REPLICAS_SERVICE_PORT = 5432 -# environment -PG_HA_HA_PORT_5432_TCP_PORT = 5432 -# environment -GETTY_CODE_CODE_SERVER_SERVICE_HOST = 10.245.62.90 -# environment -PGADMIN4_SERVICE_HOST = 10.245.182.193 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_0_PORT = tcp://10.245.135.162:8444 -# environment -SEAWEED_FILER_PORT_8888_TCP_PROTO = tcp -# environment -SEAWEED_MASTER_PORT_19333_TCP = tcp://10.245.145.57:19333 -# environment -REDIS_MASTER_SERVICE_PORT = 6379 -# environment -SEAWEED_VOLUME_0_PORT_18444_TCP_PORT = 18444 -# environment -VSCODE_HANDLES_UNCAUGHT_ERRORS = true -# default -.VARIABLES := -# environment -PWD = /home/coder/repos/seaweedfs -# automatic -%D = $(patsubst %/,%,$(dir $%)) -# environment -HOSTNAME = code-server-5dfcd6ff77-w8kdz -# environment -PG_HA_PGBOUNCER_PORT_5432_TCP = tcp://10.245.186.4:5432 -# environment -SEAWEED_VOLUME_2_PORT_8444_TCP = tcp://10.245.184.0:8444 -# environment -PG_HA_HA_SERVICE_PORT_POSTGRES = 5432 -# environment -TRAEFIK_PORT_80_TCP_ADDR = 10.245.52.194 -# environment -PG_HA_REPLICAS_PORT = tcp://10.245.76.221:5432 -# environment -SEAWEED_FILER_PORT_18888_TCP = tcp://10.245.202.58:18888 -# automatic -^D = $(patsubst %/,%,$(dir $^)) -# environment -VSCODE_LOG_STACK = false -# automatic -%F = $(notdir $%) -# environment -PASTEY_SERVICE_PORT = 5000 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_PORT = 8686 -# environment -LANG = C -# default -.LOADED := -# default -.INCLUDE_DIRS = /usr/local/include /usr/include /usr/include -# environment -PG_HA_HA_PORT_5432_TCP = tcp://10.245.23.155:5432 -# environment -TRAEFIK_PORT_22_TCP = tcp://10.245.52.194:22 -# makefile -MAKEFLAGS = pqrR -# environment -SEAWEED_VOLUME_1_SERVICE_PORT_VOLUME_HTTP = 8444 -# environment -REDIS_REPLICAS_PORT_6379_TCP_PROTO = tcp -# environment -TRAEFIK_SERVICE_PORT_WEBSECURE = 443 -# environment -REDIS_MASTER_SERVICE_PORT_TCP_REDIS = 6379 -# environment -SEAWEED_VOLUME_1_PORT_18444_TCP_PORT = 18444 -# environment -SEAWEED_VOLUME_2_SERVICE_PORT = 8444 -# makefile -CURDIR := /home/coder/repos/seaweedfs -# environment -SEAWEED_FILER_SERVICE_HOST = 10.245.202.58 -# environment -VSCODE_PIPE_LOGGING = true -# environment -SEAWEED_FILER_SERVICE_PORT_FILER_GRPC = 18888 -# environment -REDIS_REPLICAS_SERVICE_PORT = 6379 -# environment -PG_HA_PGBOUNCER_SERVICE_HOST = 10.245.186.4 -# environment -APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = true -# environment -PG_HA_REPLICAS_SERVICE_HOST = 10.245.76.221 -# automatic -*D = $(patsubst %/,%,$(dir $*)) -# environment -PGADMIN4_PORT_80_TCP_ADDR = 10.245.182.193 -# environment -SEAWEED_MASTER_PORT_9333_TCP = tcp://10.245.145.57:9333 -# environment -TRAEFIK_SERVICE_HOST = 10.245.52.194 -# default -.SHELLFLAGS := -c -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8383_TCP_PORT = 8383 -# environment -KEYCLOAK_OPERATOR_METRICS_PORT_8686_TCP_ADDR = 10.245.169.206 -# environment -MFLAGS = -pqrR -# automatic -+D = $(patsubst %/,%,$(dir $+)) -# environment -PASTEY_PORT_5000_TCP_PROTO = tcp -# makefile -MAKEFILE_LIST := -# environment -PG_HA_PGBOUNCER_SERVICE_PORT = 5432 -# environment -REDIS_MASTER_PORT_6379_TCP_ADDR = 10.245.29.137 -# environment -REDIS_REPLICAS_PORT = tcp://10.245.122.67:6379 -# environment -GETTY_CODE_CODE_SERVER_PORT_8080_TCP = tcp://10.245.62.90:8080 -# environment -PG_HA_REPLICAS_PORT_5432_TCP = tcp://10.245.76.221:5432 -# environment -SEAWEED_MASTER_PORT_9333_TCP_PROTO = tcp -# automatic -@F = $(notdir $@) -# environment -VSCODE_VERBOSE_LOGGING = true -# environment -TRAEFIK_PORT_443_TCP_PROTO = tcp -# automatic -?D = $(patsubst %/,%,$(dir $?)) -# environment -PASTEY_PORT = tcp://10.245.184.85:5000 -# environment -TRAEFIK_PORT_80_TCP_PROTO = tcp -# environment -SEAWEED_MASTER_PORT_9333_TCP_PORT = 9333 -# environment -GITEA_MEMCACHED_PORT = tcp://10.245.55.95:11211 -# environment -SEAWEED_VOLUME_0_PORT_18444_TCP_PROTO = tcp -# environment -SEAWEED_MASTER_PORT_19333_TCP_PROTO = tcp -# environment -SEAWEED_VOLUME_0_SERVICE_PORT = 8444 -# environment -PLIK_PORT_8080_TCP = tcp://10.245.176.243:8080 -# environment -KEYCLOAK_OPERATOR_METRICS_SERVICE_PORT_HTTP_METRICS = 8383 -# environment -SEAWEED_VOLUME_2_PORT_8444_TCP_ADDR = 10.245.184.0 -# environment -SEAWEED_VOLUME_2_PORT = tcp://10.245.184.0:8444 -# automatic -*F = $(notdir $*) -# environment -TRAEFIK_PORT_22_TCP_ADDR = 10.245.52.194 -# environment -KUBERNETES_PORT_443_TCP_ADDR = 10.245.0.1 -# environment -CODE_SERVER_PORT_8080_TCP = tcp://10.245.153.237:8080 -# environment -TRAEFIK_PORT_22_TCP_PORT = 22 -# automatic - Date: Mon, 28 Mar 2022 17:28:53 +0000 Subject: [PATCH 21/50] Replace with hyphen --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a17a8d7d..29f8fd342 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,4 @@ - -build_binary_latest: +build-binary-latest: stage: build image: name: golang:1.18.0-alpine3.15 @@ -8,7 +7,7 @@ build_binary_latest: rules: - if: $CI_COMMIT_TAG -build_container_latest: +build-container-latest: stage: build image: name: gcr.io/kaniko-project/executor:debug From 9f0957b4d09feffd4c587b0da4257b83e16acc52 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:42:03 +0000 Subject: [PATCH 22/50] Remove restrictions --- .gitlab-ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29f8fd342..608ab54fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,8 +4,6 @@ build-binary-latest: name: golang:1.18.0-alpine3.15 script: - go build -o ./weed/weed ./weed - rules: - - if: $CI_COMMIT_TAG build-container-latest: stage: build @@ -19,6 +17,4 @@ build-container-latest: /kaniko/executor --context "${CI_PROJECT_DIR}/docker" --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" - rules: - - if: $CI_COMMIT_TAG \ No newline at end of file + --destination "${CI_REGISTRY_IMAGE}:latest" \ No newline at end of file From c464e2e0d5d2ab0be9c6a269b1ef4f848d085803 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 17:43:03 +0000 Subject: [PATCH 23/50] Remove restrictions --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 608ab54fa..58112db6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,4 +17,5 @@ build-container-latest: /kaniko/executor --context "${CI_PROJECT_DIR}/docker" --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" - --destination "${CI_REGISTRY_IMAGE}:latest" \ No newline at end of file + --destination "${CI_REGISTRY_IMAGE}:latest" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" \ No newline at end of file From ca6becfeec7ca94736d5e8b46e78b5564dd44fe8 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:08:16 +0000 Subject: [PATCH 24/50] df changes --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58112db6c..65d88908e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,11 @@ build-binary-latest: image: name: golang:1.18.0-alpine3.15 script: - - go build -o ./weed/weed ./weed + - go build -o ./weed/weed ./weed + - cp ./weed/weed docker/weed + artifacts: + paths: + - ./weed/weed build-container-latest: stage: build @@ -16,6 +20,6 @@ build-container-latest: - >- /kaniko/executor --context "${CI_PROJECT_DIR}/docker" - --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.go_build" + --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.local" --destination "${CI_REGISTRY_IMAGE}:latest" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" \ No newline at end of file From 644a6139868393ce5acee134da8b43dc4c48c1f8 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:15:47 +0000 Subject: [PATCH 25/50] needs --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65d88908e..4d5b9674d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ build-container-latest: image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] + needs: ["build-binary-latest"] script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json From ec6320b037894b4cc023d0c5f078a2cba5c5fabe Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:16:24 +0000 Subject: [PATCH 26/50] Add gitg --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d5b9674d..d42960b33 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ build-binary-latest: image: name: golang:1.18.0-alpine3.15 script: + - apk add git - go build -o ./weed/weed ./weed - cp ./weed/weed docker/weed artifacts: From 1d47574531b8475751eb0338dd10f30a1e16d1ff Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 18:22:38 +0000 Subject: [PATCH 27/50] add build base --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d42960b33..3bbf60537 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ build-binary-latest: image: name: golang:1.18.0-alpine3.15 script: - - apk add git + - apk add git build-base - go build -o ./weed/weed ./weed - cp ./weed/weed docker/weed artifacts: From ea0b41d91c0f017cf4a30bba83f88b94197f38fc Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Mar 2022 11:24:24 -0700 Subject: [PATCH 28/50] remove --- .gitlab-ci.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 3bbf60537..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,27 +0,0 @@ -build-binary-latest: - stage: build - image: - name: golang:1.18.0-alpine3.15 - script: - - apk add git build-base - - go build -o ./weed/weed ./weed - - cp ./weed/weed docker/weed - artifacts: - paths: - - ./weed/weed - -build-container-latest: - stage: build - image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] - needs: ["build-binary-latest"] - script: - - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json - - >- - /kaniko/executor - --context "${CI_PROJECT_DIR}/docker" - --dockerfile "${CI_PROJECT_DIR}/docker/Dockerfile.local" - --destination "${CI_REGISTRY_IMAGE}:latest" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" \ No newline at end of file From 1d9e30d8c0606d4cf29a5989a12c0dad6150457b Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 31 Mar 2022 19:10:06 +0500 Subject: [PATCH 29/50] fsck replicas --- weed/shell/command_volume_fsck.go | 184 +++++++++++++++++------------- 1 file changed, 103 insertions(+), 81 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 7d3aa28a5..7318d475c 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "strings" "sync" @@ -65,8 +66,11 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. verbose := fsckCommand.Bool("v", false, "verbose mode") findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"") findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") + findMissingChunksInVolumeId := fsckCommand.Int("findMissingChunksInVolumeId", 0, "used together with findMissingChunksInFiler") applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer") purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") + tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files") + if err = fsckCommand.Parse(args); err != nil { return nil } @@ -78,7 +82,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. c.env = commandEnv // create a temp folder - tempFolder, err := os.MkdirTemp("", "sw_fsck") + tempFolder, err := os.MkdirTemp(*tempPath, "sw_fsck") if err != nil { return fmt.Errorf("failed to create temp folder: %v", err) } @@ -88,14 +92,14 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. defer os.RemoveAll(tempFolder) // collect all volume id locations - volumeIdToVInfo, err := c.collectVolumeIds(commandEnv, *verbose, writer) + dataNodeVolumeIdToVInfo, err := c.collectVolumeIds(commandEnv, *verbose, writer) if err != nil { return fmt.Errorf("failed to collect all volume locations: %v", err) } isBucketsPath := false var fillerBucketsPath string - if *findMissingChunksInFiler && *findMissingChunksInFilerPath != "" { + if *findMissingChunksInFiler && *findMissingChunksInFilerPath != "/" { fillerBucketsPath, err = readFilerBucketsPath(commandEnv) if err != nil { return fmt.Errorf("read filer buckets path: %v", err) @@ -109,33 +113,41 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. } // collect each volume file ids - for volumeId, vinfo := range volumeIdToVInfo { - if isBucketsPath && !strings.HasPrefix(*findMissingChunksInFilerPath, fillerBucketsPath+"/"+vinfo.collection) { - delete(volumeIdToVInfo, volumeId) - continue - } - err = c.collectOneVolumeFileIds(tempFolder, volumeId, vinfo, *verbose, writer) - if err != nil { - return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err) + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + for volumeId, vinfo := range volumeIdToVInfo { + if *findMissingChunksInVolumeId > 0 && uint32(*findMissingChunksInVolumeId) != volumeId { + delete(volumeIdToVInfo, volumeId) + continue + } + if isBucketsPath && !strings.HasPrefix(*findMissingChunksInFilerPath, fillerBucketsPath+"/"+vinfo.collection) { + delete(volumeIdToVInfo, volumeId) + continue + } + err = c.collectOneVolumeFileIds(tempFolder, dataNodeId, volumeId, vinfo, *verbose, writer) + if err != nil { + return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err) + } } } if *findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *purgeAbsent); err != nil { + if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *purgeAbsent); err != nil { return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) } - // for each volume, check filer file ids - if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { - return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err) + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + // for each volume, check filer file ids + if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, dataNodeId, writer, *verbose, *applyPurging); err != nil { + return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err) + } } } else { // collect all filer file ids - if err = c.collectFilerFileIds(volumeIdToVInfo, tempFolder, writer, *verbose); err != nil { + if err = c.collectFilerFileIds(dataNodeVolumeIdToVInfo, tempFolder, writer, *verbose); err != nil { return fmt.Errorf("failed to collect file ids from filer: %v", err) } // volume file ids subtract filer file ids - if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { + if err = c.findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { return fmt.Errorf("findExtraChunksInVolumeServers: %v", err) } } @@ -143,19 +155,23 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, purgeAbsent bool) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, purgeAbsent bool) error { if verbose { fmt.Fprintf(writer, "checking each file from filer ...\n") } files := make(map[uint32]*os.File) - for vid := range volumeIdToServer { - dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if openErr != nil { - return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + for _, volumeIdToServer := range dataNodeVolumeIdToVInfo { + for vid := range volumeIdToServer { + dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if openErr != nil { + return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + } + if _, ok := volumeIdToServer[vid]; !ok { + files[vid] = dst + } } - files[vid] = dst } defer func() { for _, f := range files { @@ -210,10 +226,10 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint } -func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { +func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, dataNodeId string, writer io.Writer, verbose bool, applyPurging bool) error { for volumeId, vinfo := range volumeIdToVInfo { - checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose, applyPurging) + checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, dataNodeId, volumeId, writer, verbose, applyPurging) if checkErr != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } @@ -221,55 +237,57 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf return nil } -func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { +func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 - for volumeId, vinfo := range volumeIdToVInfo { - inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer, verbose) - if checkErr != nil { - return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) - } - totalInUseCount += inUseCount - totalOrphanChunkCount += uint64(len(orphanFileIds)) - totalOrphanDataSize += orphanDataSize - - if verbose { - for _, fid := range orphanFileIds { - fmt.Fprintf(writer, "%s\n", fid) + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + for volumeId, vinfo := range volumeIdToVInfo { + inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, dataNodeId, volumeId, writer, verbose) + if checkErr != nil { + return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } - } + totalInUseCount += inUseCount + totalOrphanChunkCount += uint64(len(orphanFileIds)) + totalOrphanDataSize += orphanDataSize - if applyPurging && len(orphanFileIds) > 0 { if verbose { - fmt.Fprintf(writer, "purging process for volume %d", volumeId) + for _, fid := range orphanFileIds { + fmt.Fprintf(writer, "%s\n", fid) + } } - if vinfo.isEcVolume { - fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) - continue - } - - needleVID := needle.VolumeId(volumeId) - - if vinfo.isReadOnly { - err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, true) - if err != nil { - return fmt.Errorf("mark volume %d read/write: %v", volumeId, err) + if applyPurging && len(orphanFileIds) > 0 { + if verbose { + fmt.Fprintf(writer, "purging process for volume %d", volumeId) } - fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) - defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, false) - } + if vinfo.isEcVolume { + fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) + continue + } - fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) + needleVID := needle.VolumeId(volumeId) - if verbose { - fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) - } + if vinfo.isReadOnly { + err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, true) + if err != nil { + return fmt.Errorf("mark volume %d read/write: %v", volumeId, err) + } - if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { - return fmt.Errorf("purging volume %d: %v", volumeId, err) + fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) + defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, false) + } + + fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) + + if verbose { + fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) + } + + if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { + return fmt.Errorf("purging volume %d: %v", volumeId, err) + } } } } @@ -290,7 +308,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u return nil } -func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId uint32, vinfo VInfo, verbose bool, writer io.Writer) error { +func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeId string, volumeId uint32, vinfo VInfo, verbose bool, writer io.Writer) error { if verbose { fmt.Fprintf(writer, "collecting volume %d file ids from %s ...\n", volumeId, vinfo.server) @@ -316,7 +334,7 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err) } - err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId)) + err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)) if err != nil { return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err) } @@ -327,19 +345,21 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId } -func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool) error { +func (c *commandVolumeFsck) collectFilerFileIds(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool) error { if verbose { fmt.Fprintf(writer, "collecting file ids from filer ...\n") } files := make(map[uint32]*os.File) - for vid := range volumeIdToServer { - dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if openErr != nil { - return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + for _, volumeIdToServer := range dataNodeVolumeIdToVInfo { + for vid := range volumeIdToServer { + dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if openErr != nil { + return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) + } + files[vid] = dst } - files[vid] = dst } defer func() { for _, f := range files { @@ -377,7 +397,7 @@ func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInf }) } -func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, dataNodeId string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) { if verbose { fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId) @@ -386,7 +406,7 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo db := needle_map.NewMemDb() defer db.Close() - if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil { + if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)); err != nil { return } @@ -473,12 +493,12 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath, verbose bool) { } } -func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, dataNodeId string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { db := needle_map.NewMemDb() defer db.Close() - if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil { + if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)); err != nil { return } @@ -509,8 +529,8 @@ func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder stri if orphanFileCount > 0 { pct := float64(orphanFileCount*100) / (float64(orphanFileCount + inUseCount)) - fmt.Fprintf(writer, "volume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", - volumeId, orphanFileCount+inUseCount, orphanFileCount, pct, orphanDataSize) + fmt.Fprintf(writer, "dataNode:%s\tvolume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", + dataNodeId, volumeId, orphanFileCount+inUseCount, orphanFileCount, pct, orphanDataSize) } return @@ -524,13 +544,13 @@ type VInfo struct { isReadOnly bool } -func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose bool, writer io.Writer) (volumeIdToServer map[uint32]VInfo, err error) { +func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose bool, writer io.Writer) (volumeIdToServer map[string]map[uint32]VInfo, err error) { if verbose { fmt.Fprintf(writer, "collecting volume id and locations from master ...\n") } - volumeIdToServer = make(map[uint32]VInfo) + volumeIdToServer = make(map[string]map[uint32]VInfo) // collect topology information topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { @@ -539,8 +559,10 @@ func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose boo eachDataNode(topologyInfo, func(dc string, rack RackId, t *master_pb.DataNodeInfo) { for _, diskInfo := range t.DiskInfos { + dataNodeId := t.GetId() + volumeIdToServer[dataNodeId] = make(map[uint32]VInfo) for _, vi := range diskInfo.VolumeInfos { - volumeIdToServer[vi.Id] = VInfo{ + volumeIdToServer[dataNodeId][vi.Id] = VInfo{ server: pb.NewServerAddressFromDataNode(t), collection: vi.Collection, isEcVolume: false, @@ -548,7 +570,7 @@ func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose boo } } for _, ecShardInfo := range diskInfo.EcShardInfos { - volumeIdToServer[ecShardInfo.Id] = VInfo{ + volumeIdToServer[dataNodeId][ecShardInfo.Id] = VInfo{ server: pb.NewServerAddressFromDataNode(t), collection: ecShardInfo.Collection, isEcVolume: true, @@ -600,8 +622,8 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds [] return } -func getVolumeFileIdFile(tempFolder string, vid uint32) string { - return filepath.Join(tempFolder, fmt.Sprintf("%d.idx", vid)) +func getVolumeFileIdFile(tempFolder string, dataNodeid string, vid uint32) string { + return filepath.Join(tempFolder, fmt.Sprintf("%s_%d.idx", dataNodeid, vid)) } func getFilerFileIdFile(tempFolder string, vid uint32) string { From 3cedb21bb728add11978d6cf45e7c2d8a2e94d12 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 31 Mar 2022 21:36:10 +0500 Subject: [PATCH 30/50] skip new entities --- weed/shell/command_volume_fsck.go | 32 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 7318d475c..d02033aa3 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -5,17 +5,6 @@ import ( "context" "flag" "fmt" - "io" - "io/ioutil" - "math" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "strings" - "sync" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb" @@ -26,6 +15,17 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/needle_map" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" + "io" + "io/ioutil" + "math" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" ) func init() { @@ -112,6 +112,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return fmt.Errorf("read filer buckets path: %v", err) } + collectMtime := time.Now().Unix() // collect each volume file ids for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { for volumeId, vinfo := range volumeIdToVInfo { @@ -132,7 +133,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. if *findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *purgeAbsent); err != nil { + if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *purgeAbsent, collectMtime); err != nil { return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) } for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { @@ -155,7 +156,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, purgeAbsent bool) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, purgeAbsent bool, collectMtime int64) error { if verbose { fmt.Fprintf(writer, "checking each file from filer ...\n") @@ -195,6 +196,9 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m } dataChunks = append(dataChunks, manifestChunks...) for _, chunk := range dataChunks { + if chunk.Mtime > collectMtime { + continue + } outputChan <- &Item{ vid: chunk.Fid.VolumeId, fileKey: chunk.Fid.FileKey, @@ -400,7 +404,7 @@ func (c *commandVolumeFsck) collectFilerFileIds(dataNodeVolumeIdToVInfo map[stri func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, dataNodeId string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) { if verbose { - fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId) + fmt.Fprintf(writer, "find missing file chunks in dataNodeId %s volume %d ...\n", dataNodeId, volumeId) } db := needle_map.NewMemDb() From 3817e05dd043f024854b2c9acae6daecf2ff14b8 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Fri, 1 Apr 2022 10:17:09 +0500 Subject: [PATCH 31/50] fix collect filer files --- weed/shell/command_volume_fsck.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index d02033aa3..557c1e18e 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -165,13 +165,14 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m files := make(map[uint32]*os.File) for _, volumeIdToServer := range dataNodeVolumeIdToVInfo { for vid := range volumeIdToServer { + if _, ok := files[vid]; ok { + continue + } dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if openErr != nil { return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr) } - if _, ok := volumeIdToServer[vid]; !ok { - files[vid] = dst - } + files[vid] = dst } } defer func() { From 7f1383a41ef8011224f15fd9252aa0677af62324 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Fri, 1 Apr 2022 14:45:41 +0500 Subject: [PATCH 32/50] findExtraChunksInVolumeServers in consideration of replication --- weed/shell/command_volume_fsck.go | 68 +++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 557c1e18e..1aa33e054 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -245,13 +245,32 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 - + volumeIdOrphanFileIds := make(map[uint32]map[string]bool) + isSeveralReplicas := make(map[uint32]bool) + isEcVolumeReplicas := make(map[uint32]bool) + isReadOnlyReplicas := make(map[uint32]bool) + serverReplicas := make(map[uint32][]pb.ServerAddress) for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { for volumeId, vinfo := range volumeIdToVInfo { inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, dataNodeId, volumeId, writer, verbose) if checkErr != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } + isSeveralReplicas[volumeId] = false + if _, found := volumeIdOrphanFileIds[volumeId]; !found { + volumeIdOrphanFileIds[volumeId] = make(map[string]bool) + } else { + isSeveralReplicas[volumeId] = true + } + for _, fid := range orphanFileIds { + if isSeveralReplicas[volumeId] { + if _, found := volumeIdOrphanFileIds[volumeId][fid]; !found { + continue + } + } + volumeIdOrphanFileIds[volumeId][fid] = isSeveralReplicas[volumeId] + } + totalInUseCount += inUseCount totalOrphanChunkCount += uint64(len(orphanFileIds)) totalOrphanDataSize += orphanDataSize @@ -261,31 +280,48 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn fmt.Fprintf(writer, "%s\n", fid) } } + isEcVolumeReplicas[volumeId] = vinfo.isEcVolume + if isReadOnly, found := isReadOnlyReplicas[volumeId]; !(found && isReadOnly) { + isReadOnlyReplicas[volumeId] = vinfo.isReadOnly + } + serverReplicas[volumeId] = append(serverReplicas[volumeId], vinfo.server) + } - if applyPurging && len(orphanFileIds) > 0 { - if verbose { - fmt.Fprintf(writer, "purging process for volume %d", volumeId) - } - - if vinfo.isEcVolume { - fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) - continue + for volumeId, orphanReplicaFileIds := range volumeIdOrphanFileIds { + if !(applyPurging && len(orphanReplicaFileIds) > 0) { + continue + } + orphanFileIds := []string{} + for fid, foundInAllReplicas := range orphanReplicaFileIds { + if !isSeveralReplicas[volumeId] || (isSeveralReplicas[volumeId] && foundInAllReplicas) { + orphanFileIds = append(orphanFileIds, fid) } + } + if !(len(orphanFileIds) > 0) { + continue + } + if verbose { + fmt.Fprintf(writer, "purging process for volume %d", volumeId) + } + if isEcVolumeReplicas[volumeId] { + fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) + continue + } + for _, server := range serverReplicas[volumeId] { needleVID := needle.VolumeId(volumeId) - if vinfo.isReadOnly { - err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, true) + if isReadOnlyReplicas[volumeId] { + err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, true) if err != nil { return fmt.Errorf("mark volume %d read/write: %v", volumeId, err) } - fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) - defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, false) + fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, server) + defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, server, false) + + fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, server) } - - fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) - if verbose { fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) } From 7e925175715c57b49552a94d25d1d3dc40f1d881 Mon Sep 17 00:00:00 2001 From: a Date: Fri, 1 Apr 2022 14:09:25 -0500 Subject: [PATCH 33/50] change user and pass to username and password --- weed/command/scaffold/filer.toml | 4 ++-- weed/filer/arangodb/arangodb_store.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 6a7835de3..0a505bbdc 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -290,8 +290,8 @@ enabled = false db_name = "seaweedfs" servers=["http://localhost:8529"] # list of servers to connect to # only basic auth supported for now -user="" -pass="" +username="" +password="" # skip tls cert validation insecure_skip_verify = true diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 27ed9132b..9fd1fffb3 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -59,8 +59,8 @@ func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix store.buckets = make(map[string]driver.Collection, 3) store.databaseName = configuration.GetString(prefix + "db_name") return store.connection(configuration.GetStringSlice(prefix+"servers"), - configuration.GetString(prefix+"user"), - configuration.GetString(prefix+"pass"), + configuration.GetString(prefix+"username"), + configuration.GetString(prefix+"password"), configuration.GetBool(prefix+"insecure_skip_verify"), ) } From 2364fab927f76a79059e8ef12cec2ff3e1685a8e Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 19 Apr 2022 11:44:41 +0500 Subject: [PATCH 34/50] volume.list show only readonly, collectionPattern and volumeId --- weed/shell/command_volume_list.go | 52 +++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/weed/shell/command_volume_list.go b/weed/shell/command_volume_list.go index 9150752d5..3a5633168 100644 --- a/weed/shell/command_volume_list.go +++ b/weed/shell/command_volume_list.go @@ -7,6 +7,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "golang.org/x/exp/slices" + "path/filepath" "io" ) @@ -16,6 +17,9 @@ func init() { } type commandVolumeList struct { + collectionPattern *string + readonly *bool + volumeId *uint64 } func (c *commandVolumeList) Name() string { @@ -34,6 +38,10 @@ func (c *commandVolumeList) Do(args []string, commandEnv *CommandEnv, writer io. volumeListCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) verbosityLevel := volumeListCommand.Int("v", 5, "verbose mode: 0, 1, 2, 3, 4, 5") + c.collectionPattern = volumeListCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'") + c.readonly = volumeListCommand.Bool("readonly", false, "show only readonly") + c.volumeId = volumeListCommand.Uint64("volumeId", 0, "show only volume id") + if err = volumeListCommand.Parse(args); err != nil { return nil } @@ -44,7 +52,7 @@ func (c *commandVolumeList) Do(args []string, commandEnv *CommandEnv, writer io. return err } - writeTopologyInfo(writer, topologyInfo, volumeSizeLimitMb, *verbosityLevel) + c.writeTopologyInfo(writer, topologyInfo, volumeSizeLimitMb, *verbosityLevel) return nil } @@ -65,53 +73,71 @@ func diskInfoToString(diskInfo *master_pb.DiskInfo) string { return buf.String() } -func writeTopologyInfo(writer io.Writer, t *master_pb.TopologyInfo, volumeSizeLimitMb uint64, verbosityLevel int) statistics { +func (c *commandVolumeList) writeTopologyInfo(writer io.Writer, t *master_pb.TopologyInfo, volumeSizeLimitMb uint64, verbosityLevel int) statistics { output(verbosityLevel >= 0, writer, "Topology volumeSizeLimit:%d MB%s\n", volumeSizeLimitMb, diskInfosToString(t.DiskInfos)) slices.SortFunc(t.DataCenterInfos, func(a, b *master_pb.DataCenterInfo) bool { return a.Id < b.Id }) var s statistics for _, dc := range t.DataCenterInfos { - s = s.plus(writeDataCenterInfo(writer, dc, verbosityLevel)) + s = s.plus(c.writeDataCenterInfo(writer, dc, verbosityLevel)) } output(verbosityLevel >= 0, writer, "%+v \n", s) return s } -func writeDataCenterInfo(writer io.Writer, t *master_pb.DataCenterInfo, verbosityLevel int) statistics { + +func (c *commandVolumeList) writeDataCenterInfo(writer io.Writer, t *master_pb.DataCenterInfo, verbosityLevel int) statistics { output(verbosityLevel >= 1, writer, " DataCenter %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics slices.SortFunc(t.RackInfos, func(a, b *master_pb.RackInfo) bool { return a.Id < b.Id }) for _, r := range t.RackInfos { - s = s.plus(writeRackInfo(writer, r, verbosityLevel)) + s = s.plus(c.writeRackInfo(writer, r, verbosityLevel)) } output(verbosityLevel >= 1, writer, " DataCenter %s %+v \n", t.Id, s) return s } -func writeRackInfo(writer io.Writer, t *master_pb.RackInfo, verbosityLevel int) statistics { + +func (c *commandVolumeList) writeRackInfo(writer io.Writer, t *master_pb.RackInfo, verbosityLevel int) statistics { output(verbosityLevel >= 2, writer, " Rack %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics slices.SortFunc(t.DataNodeInfos, func(a, b *master_pb.DataNodeInfo) bool { return a.Id < b.Id }) for _, dn := range t.DataNodeInfos { - s = s.plus(writeDataNodeInfo(writer, dn, verbosityLevel)) + s = s.plus(c.writeDataNodeInfo(writer, dn, verbosityLevel)) } output(verbosityLevel >= 2, writer, " Rack %s %+v \n", t.Id, s) return s } -func writeDataNodeInfo(writer io.Writer, t *master_pb.DataNodeInfo, verbosityLevel int) statistics { + +func (c *commandVolumeList) writeDataNodeInfo(writer io.Writer, t *master_pb.DataNodeInfo, verbosityLevel int) statistics { output(verbosityLevel >= 3, writer, " DataNode %s%s\n", t.Id, diskInfosToString(t.DiskInfos)) var s statistics for _, diskInfo := range t.DiskInfos { - s = s.plus(writeDiskInfo(writer, diskInfo, verbosityLevel)) + s = s.plus(c.writeDiskInfo(writer, diskInfo, verbosityLevel)) } output(verbosityLevel >= 3, writer, " DataNode %s %+v \n", t.Id, s) return s } -func writeDiskInfo(writer io.Writer, t *master_pb.DiskInfo, verbosityLevel int) statistics { +func (c *commandVolumeList) isNotMatchDiskInfo(readOnly bool, collection string, volumeId uint32) bool { + if *c.readonly && !readOnly { + return true + } + if *c.collectionPattern != "" { + if matched, _ := filepath.Match(*c.collectionPattern, collection); !matched { + return true + } + } + if *c.volumeId > 0 && *c.volumeId != uint64(volumeId) { + return true + } + return false +} + +func (c *commandVolumeList) writeDiskInfo(writer io.Writer, t *master_pb.DiskInfo, verbosityLevel int) statistics { var s statistics diskType := t.Type if diskType == "" { @@ -122,9 +148,15 @@ func writeDiskInfo(writer io.Writer, t *master_pb.DiskInfo, verbosityLevel int) return a.Id < b.Id }) for _, vi := range t.VolumeInfos { + if c.isNotMatchDiskInfo(vi.ReadOnly, vi.Collection, vi.Id) { + continue + } s = s.plus(writeVolumeInformationMessage(writer, vi, verbosityLevel)) } for _, ecShardInfo := range t.EcShardInfos { + if c.isNotMatchDiskInfo(false, ecShardInfo.Collection, ecShardInfo.Id) { + continue + } output(verbosityLevel >= 5, writer, " ec volume id:%v collection:%v shards:%v\n", ecShardInfo.Id, ecShardInfo.Collection, erasure_coding.ShardBits(ecShardInfo.EcIndexBits).ShardIds()) } output(verbosityLevel >= 4, writer, " Disk %s %+v \n", diskType, s) From 3d5fc72d5534d169deed87b8a2a7e24870ab0aab Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Tue, 19 Apr 2022 20:04:54 -0500 Subject: [PATCH 35/50] chore(deps): Included dependency review > Dependency Review GitHub Action in your repository to enforce dependency reviews on your pull requests. > The action scans for vulnerable versions of dependencies introduced by package version changes in pull requests, > and warns you about the associated security vulnerabilities. > This gives you better visibility of what's changing in a pull request, > and helps prevent vulnerabilities being added to your repository. https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/depsreview.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/depsreview.yml diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml new file mode 100644 index 000000000..626c5d154 --- /dev/null +++ b/.github/workflows/depsreview.yml @@ -0,0 +1,14 @@ +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748 + - name: 'Dependency Review' + uses: actions/dependency-review-action@3f943b86c9a289f4e632c632695e2e0898d9d67d From 8e9ad7db5ac31ef38649a44a7599f6431ee616ad Mon Sep 17 00:00:00 2001 From: chrislu Date: Tue, 19 Apr 2022 22:58:57 -0700 Subject: [PATCH 36/50] fix bug deleting volume or unmount volume if a volume server has multiple directories --- weed/storage/store.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/weed/storage/store.go b/weed/storage/store.go index 30fe63b63..fa2897fbc 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -434,10 +434,13 @@ func (s *Store) UnmountVolume(i needle.VolumeId) error { } for _, location := range s.Locations { - if err := location.UnloadVolume(i); err == nil || err == ErrVolumeNotFound { + err := location.UnloadVolume(i) + if err == nil { glog.V(0).Infof("UnmountVolume %d", i) s.DeletedVolumesChan <- message return nil + } else if err == ErrVolumeNotFound { + continue } } @@ -458,10 +461,13 @@ func (s *Store) DeleteVolume(i needle.VolumeId) error { DiskType: string(v.location.DiskType), } for _, location := range s.Locations { - if err := location.DeleteVolume(i); err == nil || err == ErrVolumeNotFound { + err := location.DeleteVolume(i) + if err == nil { glog.V(0).Infof("DeleteVolume %d", i) s.DeletedVolumesChan <- message return nil + } else if err == ErrVolumeNotFound { + continue } else { glog.Errorf("DeleteVolume %d: %v", i, err) } From 4792e07228382666c600fa338e254a35fc7b54d4 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 20 Apr 2022 09:32:17 -0700 Subject: [PATCH 37/50] fix index out of range --- weed/iamapi/iamapi_management_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 5fea49f5c..ff6fc4c61 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -382,7 +382,7 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu // "If you do not specify a user name, IAM determines the user name implicitly based on the Amazon Web // Services access key ID signing the request." func handleImplicitUsername(r *http.Request, values url.Values) { - if values.Get("UserName") == "" { + if values.Get("UserName") == "" && len(r.Header["Authorization"]) > 0 { // get username who signs the request userName := strings.Split(r.Header["Authorization"][0], "/")[2] values.Set("UserName", userName) From 3aa4dc6ad5bbbb2eb79787345056a8fffecdc0c1 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 20 Apr 2022 09:40:24 -0700 Subject: [PATCH 38/50] Revert "fix index out of range" This reverts commit 4792e07228382666c600fa338e254a35fc7b54d4. --- weed/iamapi/iamapi_management_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index ff6fc4c61..5fea49f5c 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -382,7 +382,7 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu // "If you do not specify a user name, IAM determines the user name implicitly based on the Amazon Web // Services access key ID signing the request." func handleImplicitUsername(r *http.Request, values url.Values) { - if values.Get("UserName") == "" && len(r.Header["Authorization"]) > 0 { + if values.Get("UserName") == "" { // get username who signs the request userName := strings.Split(r.Header["Authorization"][0], "/")[2] values.Set("UserName", userName) From bdc4d67de8ffe79a4f1fd19d98f9ba7799109823 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 20 Apr 2022 09:40:41 -0700 Subject: [PATCH 39/50] Revert "Merge pull request #2944 from guo-sj/handle_implicit_username" This reverts commit bc96e73b8dfbc25f285d17ac4438c85dfd958be3, reversing changes made to 88a669dd193ce03fd4a511487e1a0fdaab118890. --- weed/iamapi/iamapi_management_handlers.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 5fea49f5c..94003c46e 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -377,18 +377,6 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu return resp } -// handleImplicitUsername adds username who signs the request to values if 'username' is not specified -// According to https://awscli.amazonaws.com/v2/documentation/api/latest/reference/iam/create-access-key.html/ -// "If you do not specify a user name, IAM determines the user name implicitly based on the Amazon Web -// Services access key ID signing the request." -func handleImplicitUsername(r *http.Request, values url.Values) { - if values.Get("UserName") == "" { - // get username who signs the request - userName := strings.Split(r.Header["Authorization"][0], "/")[2] - values.Set("UserName", userName) - } -} - func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) @@ -413,7 +401,6 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { response = iama.ListUsers(s3cfg, values) changed = false case "ListAccessKeys": - handleImplicitUsername(r, values) response = iama.ListAccessKeys(s3cfg, values) changed = false case "CreateUser": @@ -441,10 +428,8 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { return } case "CreateAccessKey": - handleImplicitUsername(r, values) response = iama.CreateAccessKey(s3cfg, values) case "DeleteAccessKey": - handleImplicitUsername(r, values) response = iama.DeleteAccessKey(s3cfg, values) case "CreatePolicy": response, err = iama.CreatePolicy(s3cfg, values) From a7df63667d8db2ee5426b160ec739896c70957fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 16:42:38 +0000 Subject: [PATCH 40/50] Bump github.com/tidwall/gjson from 1.14.0 to 1.14.1 Bumps [github.com/tidwall/gjson](https://github.com/tidwall/gjson) from 1.14.0 to 1.14.1. - [Release notes](https://github.com/tidwall/gjson/releases) - [Commits](https://github.com/tidwall/gjson/compare/v1.14.0...v1.14.1) --- updated-dependencies: - dependency-name: github.com/tidwall/gjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9201197e4..b238408d9 100644 --- a/go.mod +++ b/go.mod @@ -102,7 +102,7 @@ require ( github.com/stretchr/testify v1.7.1 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 - github.com/tidwall/gjson v1.14.0 + github.com/tidwall/gjson v1.14.1 github.com/tidwall/match v1.1.1 github.com/tidwall/pretty v1.2.0 // indirect github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 diff --git a/go.sum b/go.sum index b6461378a..f35b4d7fc 100644 --- a/go.sum +++ b/go.sum @@ -890,8 +890,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= -github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= -github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= +github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= From c57e0a98ee3be8da358aaedb50b86891367f52ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 16:44:26 +0000 Subject: [PATCH 41/50] Bump github.com/aws/aws-sdk-go from 1.43.41 to 1.43.42 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.43.41 to 1.43.42. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.43.41...v1.43.42) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b238408d9..20ff1821f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.14.0 github.com/OneOfOne/xxhash v1.2.8 github.com/Shopify/sarama v1.32.0 - github.com/aws/aws-sdk-go v1.43.41 + github.com/aws/aws-sdk-go v1.43.42 github.com/beorn7/perks v1.0.1 // indirect github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/bwmarrin/snowflake v0.3.0 diff --git a/go.sum b/go.sum index f35b4d7fc..a16c5d7f1 100644 --- a/go.sum +++ b/go.sum @@ -154,8 +154,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.43.41 h1:HaazVplP8/t6SOfybQlNUmjAxLWDKdLdX8BSEHFlJdY= -github.com/aws/aws-sdk-go v1.43.41/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.43.42 h1:y2Ay+hJnaybK6qsiV38tS+FnKJhb3F/3L/TZFxjbusM= +github.com/aws/aws-sdk-go v1.43.42/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= From cb1e7aa27df4d0b5f864799903b2cca1d5b90d92 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 20 Apr 2022 16:21:56 -0700 Subject: [PATCH 42/50] fix build --- weed/filer/arangodb/arangodb_store_bucket.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/filer/arangodb/arangodb_store_bucket.go b/weed/filer/arangodb/arangodb_store_bucket.go index 63d407309..810d639a7 100644 --- a/weed/filer/arangodb/arangodb_store_bucket.go +++ b/weed/filer/arangodb/arangodb_store_bucket.go @@ -2,9 +2,9 @@ package arangodb import ( "context" + "github.com/arangodb/go-driver" "time" - "github.com/arangodb/go-driver" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -18,7 +18,7 @@ func (store *ArangodbStore) OnBucketCreation(bucket string) { // create the collection && add to cache _, err := store.ensureBucket(timeout, bucket) if err != nil { - glog.V(0).Infof("bucket create %s : %w", bucket, err) + glog.Errorf("bucket create %s: %v", bucket, err) } } func (store *ArangodbStore) OnBucketDeletion(bucket string) { @@ -26,12 +26,12 @@ func (store *ArangodbStore) OnBucketDeletion(bucket string) { defer cancel() collection, err := store.ensureBucket(timeout, bucket) if err != nil { - glog.V(0).Infof("bucket delete %s : %w", bucket, err) + glog.Errorf("bucket delete %s: %v", bucket, err) return } err = collection.Remove(timeout) if err != nil && !driver.IsNotFound(err) { - glog.V(0).Infof("bucket delete %s : %w", bucket, err) + glog.Errorf("bucket delete %s: %v", bucket, err) return } } From 6994e8819234648c01f232a9786a2bdeb275194e Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 21 Apr 2022 00:02:18 -0700 Subject: [PATCH 43/50] fix typo --- weed/command/filer_copy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 06bb82319..9a41dd933 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -71,7 +71,7 @@ var cmdFilerCopy = &Command{ It can copy one or a list of files or folders. If copying a whole folder recursively: - All files under the folder and subfolders will be copyed. + All files under the folder and sub folders will be copied. Optional parameter "-include" allows you to specify the file name patterns. If "maxMB" is set to a positive number, files larger than it would be split into chunks. From 3885374edf80c5c07551c2a43948614b98936fe5 Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 21 Apr 2022 01:10:46 -0700 Subject: [PATCH 44/50] conditionally build elastic, gocdk to reduce binary size --- Makefile | 11 +++ weed/command/filer_meta_tail.go | 72 ---------------- weed/command/filer_meta_tail_elastic.go | 82 +++++++++++++++++++ weed/command/filer_meta_tail_non_elastic.go | 14 ++++ weed/filer/elastic/v7/doc.go | 9 ++ weed/filer/elastic/v7/elastic_store.go | 3 + weed/filer/elastic/v7/elastic_store_kv.go | 3 + weed/notification/gocdk_pub_sub/doc.go | 9 ++ .../gocdk_pub_sub/gocdk_pub_sub.go | 3 + .../sub/notification_gocdk_pub_sub.go | 3 + 10 files changed, 137 insertions(+), 72 deletions(-) create mode 100644 Makefile create mode 100644 weed/command/filer_meta_tail_elastic.go create mode 100644 weed/command/filer_meta_tail_non_elastic.go create mode 100644 weed/filer/elastic/v7/doc.go create mode 100644 weed/notification/gocdk_pub_sub/doc.go diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..844c67172 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +BINARY = weed + +SOURCE_DIR = . + +all: install + +install: + cd weed; go install + +full_install: + cd weed; go install -tags "elastic gocdk" diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index 51c4e7128..7dbeee444 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -1,12 +1,9 @@ package command import ( - "context" "fmt" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/golang/protobuf/jsonpb" - jsoniter "github.com/json-iterator/go" - elastic "github.com/olivere/elastic/v7" "os" "path/filepath" "strings" @@ -124,72 +121,3 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return true } - -type EsDocument struct { - Dir string `json:"dir,omitempty"` - Name string `json:"name,omitempty"` - IsDirectory bool `json:"isDir,omitempty"` - Size uint64 `json:"size,omitempty"` - Uid uint32 `json:"uid,omitempty"` - Gid uint32 `json:"gid,omitempty"` - UserName string `json:"userName,omitempty"` - Collection string `json:"collection,omitempty"` - Crtime int64 `json:"crtime,omitempty"` - Mtime int64 `json:"mtime,omitempty"` - Mime string `json:"mime,omitempty"` -} - -func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) { - entry := event.NewEntry - dir, name := event.NewParentPath, entry.Name - id := util.Md5String([]byte(util.NewFullPath(dir, name))) - esEntry := &EsDocument{ - Dir: dir, - Name: name, - IsDirectory: entry.IsDirectory, - Size: entry.Attributes.FileSize, - Uid: entry.Attributes.Uid, - Gid: entry.Attributes.Gid, - UserName: entry.Attributes.UserName, - Collection: entry.Attributes.Collection, - Crtime: entry.Attributes.Crtime, - Mtime: entry.Attributes.Mtime, - Mime: entry.Attributes.Mime, - } - return esEntry, id -} - -func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { - options := []elastic.ClientOptionFunc{} - options = append(options, elastic.SetURL(strings.Split(servers, ",")...)) - options = append(options, elastic.SetSniff(false)) - options = append(options, elastic.SetHealthcheck(false)) - client, err := elastic.NewClient(options...) - if err != nil { - return nil, err - } - return func(resp *filer_pb.SubscribeMetadataResponse) error { - event := resp.EventNotification - if event.OldEntry != nil && - (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) { - // delete or not update the same file - dir, name := resp.Directory, event.OldEntry.Name - id := util.Md5String([]byte(util.NewFullPath(dir, name))) - println("delete", id) - _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background()) - return err - } - if event.NewEntry != nil { - // add a new file or update the same file - esEntry, id := toEsEntry(event) - value, err := jsoniter.Marshal(esEntry) - if err != nil { - return err - } - println(string(value)) - _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background()) - return err - } - return nil - }, nil -} diff --git a/weed/command/filer_meta_tail_elastic.go b/weed/command/filer_meta_tail_elastic.go new file mode 100644 index 000000000..4c5b606a3 --- /dev/null +++ b/weed/command/filer_meta_tail_elastic.go @@ -0,0 +1,82 @@ +//go:build elastic +// +build elastic + +package command + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + jsoniter "github.com/json-iterator/go" + elastic "github.com/olivere/elastic/v7" + "strings" +) + +type EsDocument struct { + Dir string `json:"dir,omitempty"` + Name string `json:"name,omitempty"` + IsDirectory bool `json:"isDir,omitempty"` + Size uint64 `json:"size,omitempty"` + Uid uint32 `json:"uid,omitempty"` + Gid uint32 `json:"gid,omitempty"` + UserName string `json:"userName,omitempty"` + Collection string `json:"collection,omitempty"` + Crtime int64 `json:"crtime,omitempty"` + Mtime int64 `json:"mtime,omitempty"` + Mime string `json:"mime,omitempty"` +} + +func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) { + entry := event.NewEntry + dir, name := event.NewParentPath, entry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + esEntry := &EsDocument{ + Dir: dir, + Name: name, + IsDirectory: entry.IsDirectory, + Size: entry.Attributes.FileSize, + Uid: entry.Attributes.Uid, + Gid: entry.Attributes.Gid, + UserName: entry.Attributes.UserName, + Collection: entry.Attributes.Collection, + Crtime: entry.Attributes.Crtime, + Mtime: entry.Attributes.Mtime, + Mime: entry.Attributes.Mime, + } + return esEntry, id +} + +func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { + options := []elastic.ClientOptionFunc{} + options = append(options, elastic.SetURL(strings.Split(servers, ",")...)) + options = append(options, elastic.SetSniff(false)) + options = append(options, elastic.SetHealthcheck(false)) + client, err := elastic.NewClient(options...) + if err != nil { + return nil, err + } + return func(resp *filer_pb.SubscribeMetadataResponse) error { + event := resp.EventNotification + if event.OldEntry != nil && + (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) { + // delete or not update the same file + dir, name := resp.Directory, event.OldEntry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + println("delete", id) + _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background()) + return err + } + if event.NewEntry != nil { + // add a new file or update the same file + esEntry, id := toEsEntry(event) + value, err := jsoniter.Marshal(esEntry) + if err != nil { + return err + } + println(string(value)) + _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background()) + return err + } + return nil + }, nil +} diff --git a/weed/command/filer_meta_tail_non_elastic.go b/weed/command/filer_meta_tail_non_elastic.go new file mode 100644 index 000000000..f78f3ee09 --- /dev/null +++ b/weed/command/filer_meta_tail_non_elastic.go @@ -0,0 +1,14 @@ +//go:build !elastic +// +build !elastic + +package command + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { + return func(resp *filer_pb.SubscribeMetadataResponse) error { + return nil + }, nil +} diff --git a/weed/filer/elastic/v7/doc.go b/weed/filer/elastic/v7/doc.go new file mode 100644 index 000000000..704bbf6de --- /dev/null +++ b/weed/filer/elastic/v7/doc.go @@ -0,0 +1,9 @@ +/* + +Package elastic is for elastic filer store. + +The referenced "github.com/olivere/elastic/v7" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package elastic diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index a16e5ebca..cb2c66f5a 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -1,3 +1,6 @@ +//go:build elastic +// +build elastic + package elastic import ( diff --git a/weed/filer/elastic/v7/elastic_store_kv.go b/weed/filer/elastic/v7/elastic_store_kv.go index 99c03314e..43835c153 100644 --- a/weed/filer/elastic/v7/elastic_store_kv.go +++ b/weed/filer/elastic/v7/elastic_store_kv.go @@ -1,3 +1,6 @@ +//go:build elastic +// +build elastic + package elastic import ( diff --git a/weed/notification/gocdk_pub_sub/doc.go b/weed/notification/gocdk_pub_sub/doc.go new file mode 100644 index 000000000..d7fbb9f78 --- /dev/null +++ b/weed/notification/gocdk_pub_sub/doc.go @@ -0,0 +1,9 @@ +/* + +Package gocdk_pub_sub is for Azure Service Bus and RabbitMQ. + +The referenced "gocloud.dev/pubsub" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package gocdk_pub_sub diff --git a/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go b/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go index 01c4d901f..f31b6997e 100644 --- a/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go +++ b/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go @@ -1,3 +1,6 @@ +//go:build gocdk +// +build gocdk + // Package gocdk_pub_sub supports the Go CDK (Cloud Development Kit) PubSub API, // which in turn supports many providers, including Amazon SNS/SQS, Azure Service Bus, // Google Cloud PubSub, and RabbitMQ. diff --git a/weed/replication/sub/notification_gocdk_pub_sub.go b/weed/replication/sub/notification_gocdk_pub_sub.go index b16eec2e1..cb690e3ce 100644 --- a/weed/replication/sub/notification_gocdk_pub_sub.go +++ b/weed/replication/sub/notification_gocdk_pub_sub.go @@ -1,3 +1,6 @@ +//go:build gocdk +// +build gocdk + package sub import ( From 4aa39ef33f89936709c7a169ec09c7dad71c5e9f Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 21 Apr 2022 01:22:38 -0700 Subject: [PATCH 45/50] conditionally compile sqlite --- Makefile | 2 +- weed/filer/sqlite/doc.go | 9 +++++++++ weed/filer/sqlite/sqlite_store.go | 3 ++- weed/filer/sqlite/sqlite_store_unsupported.go | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 weed/filer/sqlite/doc.go diff --git a/Makefile b/Makefile index 844c67172..f324cfc2d 100644 --- a/Makefile +++ b/Makefile @@ -8,4 +8,4 @@ install: cd weed; go install full_install: - cd weed; go install -tags "elastic gocdk" + cd weed; go install -tags "elastic gocdk sqlite" diff --git a/weed/filer/sqlite/doc.go b/weed/filer/sqlite/doc.go new file mode 100644 index 000000000..833addf54 --- /dev/null +++ b/weed/filer/sqlite/doc.go @@ -0,0 +1,9 @@ +/* + +Package sqlite is for sqlite filer store. + +The referenced "modernc.org/sqlite" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package sqlite diff --git a/weed/filer/sqlite/sqlite_store.go b/weed/filer/sqlite/sqlite_store.go index ca9d38786..70a4bf390 100644 --- a/weed/filer/sqlite/sqlite_store.go +++ b/weed/filer/sqlite/sqlite_store.go @@ -1,5 +1,6 @@ -//go:build linux || darwin || windows +//go:build (linux || darwin || windows) && sqlite // +build linux darwin windows +// +build sqlite // limited GOOS due to modernc.org/libc/unistd diff --git a/weed/filer/sqlite/sqlite_store_unsupported.go b/weed/filer/sqlite/sqlite_store_unsupported.go index 0fba1ea33..351d2e501 100644 --- a/weed/filer/sqlite/sqlite_store_unsupported.go +++ b/weed/filer/sqlite/sqlite_store_unsupported.go @@ -1,5 +1,5 @@ -//go:build !linux && !darwin && !windows && !s390 && !ppc64le && !mips64 -// +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64 +//go:build !linux && !darwin && !windows && !s390 && !ppc64le && !mips64 && !sqlite +// +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64,!sqlite // limited GOOS due to modernc.org/libc/unistd From ad01c63b841dd342fee950d5ea9afb1ee6511346 Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 21 Apr 2022 01:43:01 -0700 Subject: [PATCH 46/50] conditionally skip hdfs related code --- Makefile | 2 +- weed/remote_storage/hdfs/doc.go | 9 +++++++++ weed/remote_storage/hdfs/hdfs_kerberos.go | 3 +++ weed/remote_storage/hdfs/hdfs_storage_client.go | 5 ++++- 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 weed/remote_storage/hdfs/doc.go diff --git a/Makefile b/Makefile index f324cfc2d..36da78434 100644 --- a/Makefile +++ b/Makefile @@ -8,4 +8,4 @@ install: cd weed; go install full_install: - cd weed; go install -tags "elastic gocdk sqlite" + cd weed; go install -tags "elastic gocdk sqlite hdfs" diff --git a/weed/remote_storage/hdfs/doc.go b/weed/remote_storage/hdfs/doc.go new file mode 100644 index 000000000..086c9de3f --- /dev/null +++ b/weed/remote_storage/hdfs/doc.go @@ -0,0 +1,9 @@ +/* + +Package hdfs is for remote hdfs storage. + +The referenced "github.com/colinmarc/hdfs/v2" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package hdfs diff --git a/weed/remote_storage/hdfs/hdfs_kerberos.go b/weed/remote_storage/hdfs/hdfs_kerberos.go index 50abc0ad5..ba152020a 100644 --- a/weed/remote_storage/hdfs/hdfs_kerberos.go +++ b/weed/remote_storage/hdfs/hdfs_kerberos.go @@ -1,3 +1,6 @@ +//go:build hdfs +// +build hdfs + package hdfs import ( diff --git a/weed/remote_storage/hdfs/hdfs_storage_client.go b/weed/remote_storage/hdfs/hdfs_storage_client.go index 4d76ac0ba..3b71958fd 100644 --- a/weed/remote_storage/hdfs/hdfs_storage_client.go +++ b/weed/remote_storage/hdfs/hdfs_storage_client.go @@ -1,3 +1,6 @@ +//go:build hdfs +// +build hdfs + package hdfs import ( @@ -7,7 +10,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/remote_pb" "github.com/chrislusf/seaweedfs/weed/remote_storage" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/colinmarc/hdfs/v2" + hdfs "github.com/colinmarc/hdfs/v2" "io" "os" "path" From 2988e69b1ee53b21a09e00355009c3d8fb29d127 Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 21 Apr 2022 01:43:15 -0700 Subject: [PATCH 47/50] update go build --- .github/workflows/go.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 868e3714d..96632fc86 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -32,13 +32,9 @@ jobs: - name: Get dependencies run: | cd weed; go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - name: Build - run: cd weed; go build -v . + run: cd weed; go build -tags "elastic gocdk sqlite hdfs" -v . - name: Test - run: cd weed; go test -v ./... + run: cd weed; go test -tags "elastic gocdk sqlite hdfs" -v ./... From 350d80d907a4673dabc4b6fb0d9febced12ad86b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 15:39:37 +0000 Subject: [PATCH 48/50] Bump github.com/arangodb/go-driver from 1.2.1 to 1.3.1 Bumps [github.com/arangodb/go-driver](https://github.com/arangodb/go-driver) from 1.2.1 to 1.3.1. - [Release notes](https://github.com/arangodb/go-driver/releases) - [Changelog](https://github.com/arangodb/go-driver/blob/master/CHANGELOG.md) - [Commits](https://github.com/arangodb/go-driver/compare/v1.2.1...v1.3.1) --- updated-dependencies: - dependency-name: github.com/arangodb/go-driver dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 35a130056..78fd44522 100644 --- a/go.mod +++ b/go.mod @@ -150,7 +150,7 @@ require ( require ( github.com/Jille/raft-grpc-transport v1.2.0 - github.com/arangodb/go-driver v1.2.1 + github.com/arangodb/go-driver v1.3.1 github.com/fluent/fluent-logger-golang v1.9.0 github.com/hanwen/go-fuse/v2 v2.1.0 github.com/hashicorp/raft v1.3.7 diff --git a/go.sum b/go.sum index cd5f2fb2d..7b5bc8924 100644 --- a/go.sum +++ b/go.sum @@ -141,8 +141,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8= -github.com/arangodb/go-driver v1.2.1/go.mod h1:zdDkJJnCj8DAkfbtIjIXnsTrWIiy6VhP3Vy14p+uQeY= +github.com/arangodb/go-driver v1.3.1 h1:ypwg9uwahiUekuwdDOttoLR7F5DmK5BzpSXt92poCyQ= +github.com/arangodb/go-driver v1.3.1/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= From 3f0a8543a83386b6b1a377ffaf579f5175c6fe29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 15:40:06 +0000 Subject: [PATCH 49/50] Bump google.golang.org/api from 0.74.0 to 0.75.0 Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.74.0 to 0.75.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.74.0...v0.75.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 35a130056..9284bc3f3 100644 --- a/go.mod +++ b/go.mod @@ -130,9 +130,9 @@ require ( golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect - google.golang.org/api v0.74.0 + google.golang.org/api v0.75.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect + google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect google.golang.org/grpc v1.45.0 google.golang.org/protobuf v1.28.0 gopkg.in/inf.v0 v0.9.1 // indirect @@ -158,7 +158,7 @@ require ( ) require ( - cloud.google.com/go/compute v1.5.0 // indirect + cloud.google.com/go/compute v1.6.0 // indirect cloud.google.com/go/iam v0.3.0 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/armon/go-metrics v0.3.10 // indirect diff --git a/go.sum b/go.sum index cd5f2fb2d..541797751 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,9 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0 h1:XdQIN5mdPTSBVwSIVDuY5e8ZzVAccsHvD3qTEz4zIps= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -1393,8 +1394,9 @@ google.golang.org/api v0.68.0/go.mod h1:sOM8pTpwgflXRhz+oC8H2Dr+UcbMqkPPWNJo88Q7 google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7C80= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0 h1:0AYh/ae6l9TDUvIQrDw5QRpM100P6oHgD+o3dYHMzJg= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1490,8 +1492,10 @@ google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac h1:qSNTkEN+L2mvWcLgJOR+8bdHX9rN/IdU3A1Ghpfb1Rg= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 h1:myaecH64R0bIEDjNORIel4iXubqzaHU1K2z8ajBwWcM= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From 7f049e32fe43ae27fd3650da26cee9464e2d202f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 15:40:28 +0000 Subject: [PATCH 50/50] Bump github.com/aws/aws-sdk-go from 1.43.42 to 1.43.43 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.43.42 to 1.43.43. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.43.42...v1.43.43) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 35a130056..d5646c6ca 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.14.0 github.com/OneOfOne/xxhash v1.2.8 github.com/Shopify/sarama v1.32.0 - github.com/aws/aws-sdk-go v1.43.42 + github.com/aws/aws-sdk-go v1.43.43 github.com/beorn7/perks v1.0.1 // indirect github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/bwmarrin/snowflake v0.3.0 diff --git a/go.sum b/go.sum index cd5f2fb2d..3775958e4 100644 --- a/go.sum +++ b/go.sum @@ -158,8 +158,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.43.42 h1:y2Ay+hJnaybK6qsiV38tS+FnKJhb3F/3L/TZFxjbusM= -github.com/aws/aws-sdk-go v1.43.42/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.43.43 h1:1L06qzQvl4aC3Skfh5rV7xVhGHjIZoHcqy16NoyQ1o4= +github.com/aws/aws-sdk-go v1.43.43/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=