From 275c3bb19cda3492ca987c24cd5a25c267a1eb46 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 1 May 2022 17:28:58 +0500 Subject: [PATCH 01/18] ydb initial https://github.com/chrislusf/seaweedfs/issues/2942 --- docker/compose/local-ydb-compose.yml | 30 ++++ weed/filer/ydb/readme.md | 27 +++ weed/filer/ydb/ydb_store.go | 256 +++++++++++++++++++++++++++ weed/filer/ydb/ydb_types.go | 14 ++ weed/filer/ydb/ydb_types_ydbgen.go | 194 ++++++++++++++++++++ 5 files changed, 521 insertions(+) create mode 100644 docker/compose/local-ydb-compose.yml create mode 100644 weed/filer/ydb/readme.md create mode 100644 weed/filer/ydb/ydb_store.go create mode 100644 weed/filer/ydb/ydb_types.go create mode 100644 weed/filer/ydb/ydb_types_ydbgen.go diff --git a/docker/compose/local-ydb-compose.yml b/docker/compose/local-ydb-compose.yml new file mode 100644 index 000000000..ce3e7e9ed --- /dev/null +++ b/docker/compose/local-ydb-compose.yml @@ -0,0 +1,30 @@ +version: '2' + +services: + ydb: + image: cr.yandex/yc/yandex-docker-local-ydb + ports: + - 2135:2135 + - 8765:8765 + - 2136:2136 + volumes: + - ./seaweedfs.sql:/docker-entrypoint-initdb.d/seaweedfs.sql + environment: + - YDB_DEFAULT_LOG_LEVEL=NOTICE + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 + server: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + - 8084:8080 + - 18084:18080 + - 8888:8888 + - 18888:18888 + command: "server -ip=server -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" + volumes: + - ./master-cloud.toml:/etc/seaweedfs/master.toml + depends_on: + - ydb \ No newline at end of file diff --git a/weed/filer/ydb/readme.md b/weed/filer/ydb/readme.md new file mode 100644 index 000000000..90d7a18e9 --- /dev/null +++ b/weed/filer/ydb/readme.md @@ -0,0 +1,27 @@ +## YDB + +database: https://github.com/ydb-platform/ydb +go driver: https://github.com/ydb-platform/ydb-go-sdk + +options: + +``` +[ydb] +enabled=true +db_name="seaweedfs" +servers=["http://localhost:8529"] +#basic auth +user="root" +pass="test" + +# tls settings +insecure_skip_verify=true +``` + +get ydb types +``` +ydbgen -dir weed/filer/ydb +``` + +i test using this dev database: +`make dev_ydb` diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go new file mode 100644 index 000000000..7278c6301 --- /dev/null +++ b/weed/filer/ydb/ydb_store.go @@ -0,0 +1,256 @@ +package ydb + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/yandex-cloud/ydb-go-sdk/v2" + "github.com/yandex-cloud/ydb-go-sdk/v2/connect" + "github.com/yandex-cloud/ydb-go-sdk/v2/table" + "strings" + "time" +) + +var ( + roTX = table.TxControl( + table.BeginTx(table.WithOnlineReadOnly()), + table.CommitTx(), + ) + rwTX = table.TxControl( + table.BeginTx(table.WithSerializableReadWrite()), + table.CommitTx(), + ) +) + +const ( + createQuery = ` + PRAGMA TablePathPrefix("%s"); + CREATE TABLE file_meta ( + dir_hash int64, + name Utf8, + directory Utf8, + meta String, + PRIMARY KEY (dir_hash, name) + );` + insertQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + DECLARE $directory AS Utf8; + DECLARE $meta AS String; + + UPSERT INTO file_meta + (dir_hash, name, directory, meta) + VALUES + ($dir_hash, $name, $directory, $meta);` + updateQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + DECLARE $directory AS Utf8; + DECLARE $meta AS String; + + REPLACE INTO file_meta + (dir_hash, name, directory, meta) + VALUES + ($dir_hash, $name, $directory, $meta) + COMMIT;` + deleteQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + + DELETE FROM file_meta + WHERE dir_hash == $dir_hash AND name == $name; + COMMIT;` + findQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + + SELECT meta + FROM file_meta + WHERE dir_hash == $dir_hash AND name == $name;` +) + +type YdbStore struct { + SupportBucketTable bool + DB *connect.Connection + connParams connect.ConnectParams + connCtx context.Context + dirBuckets string + tablePathPrefix string +} + +func init() { + filer.Stores = append(filer.Stores, &YdbStore{}) +} + +func (store *YdbStore) GetName() string { + return "ydb" +} + +func (store *YdbStore) Initialize(configuration util.Configuration, prefix string) (err error) { + return store.initialize(configuration.GetString(prefix + "coonectionUrl")) +} + +func (store *YdbStore) initialize(sqlUrl string) (err error) { + store.SupportBucketTable = false + var cancel context.CancelFunc + store.connCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + store.connParams = connect.MustConnectionString(sqlUrl) + store.DB, err = connect.New(store.connCtx, store.connParams) + if err != nil { + store.DB.Close() + store.DB = nil + return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err) + } + defer store.DB.Close() + + if err = store.DB.EnsurePathExists(store.connCtx, store.connParams.Database()); err != nil { + return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + } + return nil +} + +func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Entry, query string) (err error) { + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + fileMeta := FileMeta{util.HashStringToLong(dir), name, dir, meta} + return table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), query)) + if err != nil { + return err + } + _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) + return err + }), + ) +} + +func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { + return store.insertOrUpdateEntry(ctx, entry, insertQuery) +} + +func (store *YdbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { + return store.insertOrUpdateEntry(ctx, entry, updateQuery) +} + +func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { + dir, name := fullpath.DirAndName() + var res *table.Result + err = table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), findQuery)) + if err != nil { + return err + } + _, res, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$name", ydb.UTF8Value(name)))) + return err + }), + ) + if err != nil { + return nil, err + } + for res.NextSet() { + for res.NextRow() { + res.SeekItem("meta") + entry.FullPath = fullpath + if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(res.String())); err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + return entry, nil + } + } + + return nil, filer_pb.ErrNotFound +} + +func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { + dir, name := fullpath.DirAndName() + return table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteQuery)) + if err != nil { + return err + } + _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$name", ydb.UTF8Value(name)))) + return err + }), + ) +} + +func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { + return nil +} + +func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) BeginTransaction(ctx context.Context) (context.Context, error) { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) CommitTransaction(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) RollbackTransaction(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) Shutdown() { + //TODO implement me + panic("implement me") +} + +func (store *YdbStore) getPrefix(dir string) string { + prefixBuckets := store.dirBuckets + "/" + if strings.HasPrefix(dir, prefixBuckets) { + // detect bucket + bucketAndDir := dir[len(prefixBuckets):] + if t := strings.Index(bucketAndDir, "/"); t > 0 { + return bucketAndDir[:t] + } + } + return "" +} + +func (store *YdbStore) withPragma(prefix, query string) string { + if store.tablePathPrefix != "" && store.tablePathPrefix != "/" { + prefix = store.tablePathPrefix + "/" + prefix + } + return `PRAGMA TablePathPrefix("` + prefix + `");` + query +} diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go new file mode 100644 index 000000000..5bc0d2afd --- /dev/null +++ b/weed/filer/ydb/ydb_types.go @@ -0,0 +1,14 @@ +package ydb + +//go:generate ydbgen + +//ydb:gen +type FileMeta struct { + DirHash int64 `ydb:"type:int64"` + Name string `ydb:"type:utf8"` + Directory string `ydb:"type:utf8"` + Meta []byte `ydb:"-"` +} + +//ydb:gen scan,value +type FileMetas []FileMeta diff --git a/weed/filer/ydb/ydb_types_ydbgen.go b/weed/filer/ydb/ydb_types_ydbgen.go new file mode 100644 index 000000000..8bd1906f9 --- /dev/null +++ b/weed/filer/ydb/ydb_types_ydbgen.go @@ -0,0 +1,194 @@ +// Code generated by ydbgen; DO NOT EDIT. + +package ydb + +import ( + "strconv" + + "github.com/yandex-cloud/ydb-go-sdk/v2" + "github.com/yandex-cloud/ydb-go-sdk/v2/table" +) + +var ( + _ = strconv.Itoa + _ = ydb.StringValue + _ = table.NewQueryParameters +) + +func (f *FileMeta) Scan(res *table.Result) (err error) { + res.SeekItem("dir_hash") + f.DirHash = res.Int64() + + res.SeekItem("name") + f.Name = res.UTF8() + + res.SeekItem("directory") + f.Directory = res.UTF8() + + return res.Err() +} + +func (f *FileMeta) QueryParameters() *table.QueryParameters { + var v0 ydb.Value + { + vp0 := ydb.Int64Value(f.DirHash) + v0 = vp0 + } + var v1 ydb.Value + { + vp0 := ydb.UTF8Value(f.Name) + v1 = vp0 + } + var v2 ydb.Value + { + vp0 := ydb.UTF8Value(f.Directory) + v2 = vp0 + } + return table.NewQueryParameters( + table.ValueParam("$dir_hash", v0), + table.ValueParam("$name", v1), + table.ValueParam("$directory", v2), + ) +} + +func (f *FileMeta) StructValue() ydb.Value { + var v0 ydb.Value + { + var v1 ydb.Value + { + vp0 := ydb.Int64Value(f.DirHash) + v1 = vp0 + } + var v2 ydb.Value + { + vp0 := ydb.UTF8Value(f.Name) + v2 = vp0 + } + var v3 ydb.Value + { + vp0 := ydb.UTF8Value(f.Directory) + v3 = vp0 + } + v0 = ydb.StructValue( + ydb.StructFieldValue("dir_hash", v1), + ydb.StructFieldValue("name", v2), + ydb.StructFieldValue("directory", v3), + ) + } + return v0 +} + +func (f *FileMeta) StructType() ydb.Type { + var t0 ydb.Type + { + fs0 := make([]ydb.StructOption, 3) + var t1 ydb.Type + { + tp0 := ydb.TypeInt64 + t1 = tp0 + } + fs0[0] = ydb.StructField("dir_hash", t1) + var t2 ydb.Type + { + tp0 := ydb.TypeUTF8 + t2 = tp0 + } + fs0[1] = ydb.StructField("name", t2) + var t3 ydb.Type + { + tp0 := ydb.TypeUTF8 + t3 = tp0 + } + fs0[2] = ydb.StructField("directory", t3) + t0 = ydb.Struct(fs0...) + } + return t0 +} + +func (fs *FileMetas) Scan(res *table.Result) (err error) { + for res.NextRow() { + var x0 FileMeta + res.SeekItem("dir_hash") + x0.DirHash = res.Int64() + + res.SeekItem("name") + x0.Name = res.UTF8() + + res.SeekItem("directory") + x0.Directory = res.UTF8() + + if res.Err() == nil { + *fs = append(*fs, x0) + } + } + return res.Err() +} + +func (fs FileMetas) ListValue() ydb.Value { + var list0 ydb.Value + vs0 := make([]ydb.Value, len(fs)) + for i0, item0 := range fs { + var v0 ydb.Value + { + var v1 ydb.Value + { + var v2 ydb.Value + { + vp0 := ydb.Int64Value(item0.DirHash) + v2 = vp0 + } + var v3 ydb.Value + { + vp0 := ydb.UTF8Value(item0.Name) + v3 = vp0 + } + var v4 ydb.Value + { + vp0 := ydb.UTF8Value(item0.Directory) + v4 = vp0 + } + v1 = ydb.StructValue( + ydb.StructFieldValue("dir_hash", v2), + ydb.StructFieldValue("name", v3), + ydb.StructFieldValue("directory", v4), + ) + } + v0 = v1 + } + vs0[i0] = v0 + } + if len(vs0) == 0 { + var t1 ydb.Type + { + var t2 ydb.Type + { + fs0 := make([]ydb.StructOption, 3) + var t3 ydb.Type + { + tp0 := ydb.TypeInt64 + t3 = tp0 + } + fs0[0] = ydb.StructField("dir_hash", t3) + var t4 ydb.Type + { + tp0 := ydb.TypeUTF8 + t4 = tp0 + } + fs0[1] = ydb.StructField("name", t4) + var t5 ydb.Type + { + tp0 := ydb.TypeUTF8 + t5 = tp0 + } + fs0[2] = ydb.StructField("directory", t5) + t2 = ydb.Struct(fs0...) + } + t1 = t2 + } + t0 := ydb.List(t1) + list0 = ydb.ZeroValue(t0) + } else { + list0 = ydb.ListValue(vs0...) + } + return list0 +} From 6a052f6ff222032ca9c089c97f1bb1b50a2e6ac3 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 1 May 2022 20:33:03 +0500 Subject: [PATCH 02/18] ydb Sql interface --- go.mod | 2 + go.sum | 6 ++ weed/filer/ydb/ydb_queries.go | 72 ++++++++++++++++ weed/filer/ydb/ydb_store.go | 156 ++++++++++++++++++++-------------- 4 files changed, 171 insertions(+), 65 deletions(-) create mode 100644 weed/filer/ydb/ydb_queries.go diff --git a/go.mod b/go.mod index bdebd73a8..5dbd0c6d7 100644 --- a/go.mod +++ b/go.mod @@ -203,6 +203,8 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/tinylib/msgp v1.1.6 // indirect + github.com/yandex-cloud/ydb-go-sdk/v2 v2.12.0 // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect go.uber.org/atomic v1.9.0 // indirect diff --git a/go.sum b/go.sum index 96d98d9f0..4f0276f94 100644 --- a/go.sum +++ b/go.sum @@ -375,6 +375,7 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -936,6 +937,11 @@ github.com/xdg-go/scram v1.1.0 h1:d70R37I0HrDLsafRrMBXyrD4lmQbCHE873t00Vr0gm0= github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/yandex-cloud/go-genproto v0.0.0-20210809082946-a97da516c588/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= +github.com/yandex-cloud/ydb-go-sdk/v2 v2.12.0 h1:TjMLCV3Poata+0Nw+Oa/ztEFagFrochAkGg91PG49jk= +github.com/yandex-cloud/ydb-go-sdk/v2 v2.12.0/go.mod h1:L09Rymwmcpp9QMCbfGAlMtJ4kLFFsbGjkP3qv58r2yY= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc h1:xvTP0fhYNm+Ws+xC34jzF9EdorPUKkucJr0TyybqVSk= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go new file mode 100644 index 000000000..57b282a7a --- /dev/null +++ b/weed/filer/ydb/ydb_queries.go @@ -0,0 +1,72 @@ +package ydb + +const ( + createQuery = ` + PRAGMA TablePathPrefix("%s"); + CREATE TABLE file_meta ( + dir_hash int64, + name Utf8, + directory Utf8, + meta String, + PRIMARY KEY (dir_hash, name) + );` + + insertQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + DECLARE $directory AS Utf8; + DECLARE $meta AS String; + + UPSERT INTO file_meta + (dir_hash, name, directory, meta) + VALUES + ($dir_hash, $name, $directory, $meta);` + + updateQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + DECLARE $directory AS Utf8; + DECLARE $meta AS String; + + REPLACE INTO file_meta + (dir_hash, name, directory, meta) + VALUES + ($dir_hash, $name, $directory, $meta) + COMMIT;` + + deleteQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + + DELETE FROM file_meta + WHERE dir_hash == $dir_hash AND name == $name; + COMMIT;` + + findQuery = ` + DECLARE $dir_hash int64; + DECLARE $name AS Utf8; + + SELECT meta + FROM file_meta + WHERE dir_hash == $dir_hash AND name == $name;` + + deleteFolderChildrenQuery = ` + DECLARE $dir_hash int64; + DECLARE $directory AS Utf8; + + DELETE FROM file_meta + WHERE dir_hash == $dir_hash AND directory == $directory; + COMMIT;` + + ListDirectoryQuery = ` + DECLARE $dir_hash int64; + DECLARE $directory AS Utf8; + DECLARE $start_name AS Utf8; + DECLARE $prefix AS Utf8; + DECLARE $limit AS int64; + + SELECT name, meta + FROM file_meta + WHERE dir_hash == $dir_hash AND directory == $directory and name %v $start_name and name LIKE '$prefix%' + ORDER BY name ASC LIMIT $limit;` +) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 7278c6301..48017bf6f 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -4,11 +4,13 @@ import ( "context" "fmt" "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" "github.com/yandex-cloud/ydb-go-sdk/v2" "github.com/yandex-cloud/ydb-go-sdk/v2/connect" "github.com/yandex-cloud/ydb-go-sdk/v2/table" + "path" "strings" "time" ) @@ -24,53 +26,6 @@ var ( ) ) -const ( - createQuery = ` - PRAGMA TablePathPrefix("%s"); - CREATE TABLE file_meta ( - dir_hash int64, - name Utf8, - directory Utf8, - meta String, - PRIMARY KEY (dir_hash, name) - );` - insertQuery = ` - DECLARE $dir_hash int64; - DECLARE $name AS Utf8; - DECLARE $directory AS Utf8; - DECLARE $meta AS String; - - UPSERT INTO file_meta - (dir_hash, name, directory, meta) - VALUES - ($dir_hash, $name, $directory, $meta);` - updateQuery = ` - DECLARE $dir_hash int64; - DECLARE $name AS Utf8; - DECLARE $directory AS Utf8; - DECLARE $meta AS String; - - REPLACE INTO file_meta - (dir_hash, name, directory, meta) - VALUES - ($dir_hash, $name, $directory, $meta) - COMMIT;` - deleteQuery = ` - DECLARE $dir_hash int64; - DECLARE $name AS Utf8; - - DELETE FROM file_meta - WHERE dir_hash == $dir_hash AND name == $name; - COMMIT;` - findQuery = ` - DECLARE $dir_hash int64; - DECLARE $name AS Utf8; - - SELECT meta - FROM file_meta - WHERE dir_hash == $dir_hash AND name == $name;` -) - type YdbStore struct { SupportBucketTable bool DB *connect.Connection @@ -148,7 +103,7 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e if err != nil { return err } - _, res, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + _, res, err = stmt.Execute(ctx, roTX, table.NewQueryParameters( table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), table.ValueParam("$name", ydb.UTF8Value(name)))) return err @@ -157,6 +112,8 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e if err != nil { return nil, err } + defer res.Close() + for res.NextSet() { for res.NextRow() { res.SeekItem("meta") @@ -188,32 +145,105 @@ func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) } func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { - return nil + dir, _ := fullpath.DirAndName() + return table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteFolderChildrenQuery)) + if err != nil { + return err + } + _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$directory", ydb.UTF8Value(dir)))) + return err + }), + ) } func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - //TODO implement me - panic("implement me") + return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", nil) } func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - //TODO implement me - panic("implement me") + dir := string(dirPath) + var res *table.Result + startFileCompOp := ">" + if includeStartFile { + startFileCompOp = ">=" + } + err = table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), fmt.Sprintf(ListDirectoryQuery, startFileCompOp))) + if err != nil { + return err + } + _, res, err = stmt.Execute(ctx, roTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$directory", ydb.UTF8Value(dir)), + table.ValueParam("$start_name", ydb.UTF8Value(startFileName)), + table.ValueParam("$prefix", ydb.UTF8Value(prefix)), + table.ValueParam("$limit", ydb.Int64Value(limit)), + )) + return err + }), + ) + if err != nil { + return lastFileName, err + } + defer res.Close() + + for res.NextSet() { + for res.NextRow() { + res.SeekItem("name") + name := res.UTF8() + res.SeekItem("meta") + data := res.String() + if res.Err() != nil { + glog.V(0).Infof("scan %s : %v", dirPath, err) + return lastFileName, fmt.Errorf("scan %s: %v", dirPath, err) + } + lastFileName = name + + entry := &filer.Entry{ + FullPath: util.NewFullPath(dir, name), + } + if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { + glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) + return lastFileName, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) + } + + if !eachEntryFunc(entry) { + break + } + } + } + return lastFileName, nil } func (store *YdbStore) BeginTransaction(ctx context.Context) (context.Context, error) { - //TODO implement me - panic("implement me") + session, err := store.DB.Table().Pool().Create(ctx) + if err != nil { + return ctx, err + } + tx, err := session.BeginTransaction(ctx, table.TxSettings(table.WithSerializableReadWrite())) + if err != nil { + return ctx, err + } + return context.WithValue(ctx, "tx", tx), nil } func (store *YdbStore) CommitTransaction(ctx context.Context) error { - //TODO implement me - panic("implement me") + if tx, ok := ctx.Value("tx").(*table.Transaction); ok { + return tx.Commit(ctx) + } + return nil } func (store *YdbStore) RollbackTransaction(ctx context.Context) error { - //TODO implement me - panic("implement me") + if tx, ok := ctx.Value("tx").(*table.Transaction); ok { + return tx.Rollback(ctx) + } + return nil } func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { @@ -232,8 +262,7 @@ func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { } func (store *YdbStore) Shutdown() { - //TODO implement me - panic("implement me") + store.DB.Close() } func (store *YdbStore) getPrefix(dir string) string { @@ -249,8 +278,5 @@ func (store *YdbStore) getPrefix(dir string) string { } func (store *YdbStore) withPragma(prefix, query string) string { - if store.tablePathPrefix != "" && store.tablePathPrefix != "/" { - prefix = store.tablePathPrefix + "/" + prefix - } - return `PRAGMA TablePathPrefix("` + prefix + `");` + query + return `PRAGMA TablePathPrefix("` + path.Join(store.tablePathPrefix, prefix) + `");` + query } From ec0ed41e375d99ffc7d6a4290e92470c7eabc8e7 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 1 May 2022 21:20:37 +0500 Subject: [PATCH 03/18] ydb kv interface --- .../abstract_sql/abstract_sql_store_kv.go | 8 +-- weed/filer/ydb/ydb_store.go | 15 ---- weed/filer/ydb/ydb_store_kv.go | 72 +++++++++++++++++++ 3 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 weed/filer/ydb/ydb_store_kv.go diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index 03b016c76..aaf1c196c 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -18,7 +18,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by return fmt.Errorf("findDB: %v", err) } - dirStr, dirHash, name := genDirAndName(key) + dirStr, dirHash, name := GenDirAndName(key) res, err := db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value) if err == nil { @@ -53,7 +53,7 @@ func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []b return nil, fmt.Errorf("findDB: %v", err) } - dirStr, dirHash, name := genDirAndName(key) + dirStr, dirHash, name := GenDirAndName(key) row := db.QueryRowContext(ctx, store.GetSqlFind(DEFAULT_TABLE), dirHash, name, dirStr) err = row.Scan(&value) @@ -76,7 +76,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er return fmt.Errorf("findDB: %v", err) } - dirStr, dirHash, name := genDirAndName(key) + dirStr, dirHash, name := GenDirAndName(key) res, err := db.ExecContext(ctx, store.GetSqlDelete(DEFAULT_TABLE), dirHash, name, dirStr) if err != nil { @@ -92,7 +92,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er } -func genDirAndName(key []byte) (dirStr string, dirHash int64, name string) { +func GenDirAndName(key []byte) (dirStr string, dirHash int64, name string) { for len(key) < 8 { key = append(key, 0) } diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 48017bf6f..aedc11ec5 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -246,21 +246,6 @@ func (store *YdbStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - //TODO implement me - panic("implement me") -} - -func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - //TODO implement me - panic("implement me") -} - -func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { - //TODO implement me - panic("implement me") -} - func (store *YdbStore) Shutdown() { store.DB.Close() } diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go new file mode 100644 index 000000000..3473d756d --- /dev/null +++ b/weed/filer/ydb/ydb_store_kv.go @@ -0,0 +1,72 @@ +package ydb + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" + "github.com/yandex-cloud/ydb-go-sdk/v2" + "github.com/yandex-cloud/ydb-go-sdk/v2/table" +) + +func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + dirStr, dirHash, name := abstract_sql.GenDirAndName(key) + fileMeta := FileMeta{dirHash, name, dirStr, value} + return table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), insertQuery)) + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) + return fmt.Errorf("kv put: %v", err) + }), + ) +} + +func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + dirStr, dirHash, name := abstract_sql.GenDirAndName(key) + var res *table.Result + err = table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), findQuery)) + if err != nil { + return err + } + _, res, err = stmt.Execute(ctx, roTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", ydb.Int64Value(dirHash)), + table.ValueParam("$name", ydb.UTF8Value(name)))) + return err + }), + ) + if err != nil { + return nil, fmt.Errorf("kv get: %v", err) + } + defer res.Close() + + for res.NextResultSet(ctx) { + for res.NextRow() { + if err = res.Scan(&value); err != nil { + return nil, fmt.Errorf("kv get: %v", err) + } + return + } + } + return nil, filer.ErrKvNotFound +} + +func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { + dirStr, dirHash, name := abstract_sql.GenDirAndName(key) + return table.Retry(ctx, store.DB.Table().Pool(), + table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), deleteQuery)) + if err != nil { + return fmt.Errorf("kv delete: %s", err) + } + _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", ydb.Int64Value(dirHash)), + table.ValueParam("$name", ydb.UTF8Value(name)))) + return fmt.Errorf("kv delete: %s", err) + }), + ) +} From 21033ff4c3fcf1516cab32a0b955a40f3e9119d9 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 1 May 2022 22:28:55 +0500 Subject: [PATCH 04/18] refactor use const CountEntryChunksForGzip --- weed/filer/abstract_sql/abstract_sql_store.go | 2 +- weed/filer/arangodb/arangodb_store.go | 4 +- weed/filer/cassandra/cassandra_store.go | 2 +- weed/filer/etcd/etcd_store.go | 2 +- weed/filer/filerstore.go | 2 + weed/filer/hbase/hbase_store.go | 2 +- weed/filer/leveldb/leveldb_store.go | 2 +- weed/filer/leveldb2/leveldb2_store.go | 2 +- weed/filer/leveldb3/leveldb3_store.go | 2 +- weed/filer/mongodb/mongodb_store.go | 2 +- weed/filer/redis/universal_redis_store.go | 2 +- weed/filer/redis2/universal_redis_store.go | 2 +- weed/filer/redis3/universal_redis_store.go | 2 +- weed/filer/redis_lua/universal_redis_store.go | 2 +- weed/filer/ydb/readme.md | 11 ++--- weed/filer/ydb/ydb_queries.go | 2 +- weed/filer/ydb/ydb_store.go | 48 ++++++++++++++----- 17 files changed, 55 insertions(+), 36 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 4bf9b16fa..13268b944 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -156,7 +156,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } diff --git a/weed/filer/arangodb/arangodb_store.go b/weed/filer/arangodb/arangodb_store.go index 9fd1fffb3..13d14b2b0 100644 --- a/weed/filer/arangodb/arangodb_store.go +++ b/weed/filer/arangodb/arangodb_store.go @@ -157,7 +157,7 @@ func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } model := &Model{ @@ -196,7 +196,7 @@ func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } model := &Model{ diff --git a/weed/filer/cassandra/cassandra_store.go b/weed/filer/cassandra/cassandra_store.go index fb61b0771..d8c094a45 100644 --- a/weed/filer/cassandra/cassandra_store.go +++ b/weed/filer/cassandra/cassandra_store.go @@ -100,7 +100,7 @@ func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer.Entry return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } diff --git a/weed/filer/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go index 2a5dfc926..4146a3899 100644 --- a/weed/filer/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -82,7 +82,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = weed_util.MaybeGzipData(meta) } diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index a092ee456..260945b33 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -7,6 +7,8 @@ import ( "io" ) +const CountEntryChunksForGzip = 50 + var ( ErrUnsupportedListDirectoryPrefixed = errors.New("unsupported directory prefix listing") ErrUnsupportedSuperLargeDirectoryListing = errors.New("unsupported super large directory listing") diff --git a/weed/filer/hbase/hbase_store.go b/weed/filer/hbase/hbase_store.go index e0d878ca7..c5d6eb48c 100644 --- a/weed/filer/hbase/hbase_store.go +++ b/weed/filer/hbase/hbase_store.go @@ -75,7 +75,7 @@ func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) er if err != nil { return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } diff --git a/weed/filer/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go index 73d757e62..6abb37f99 100644 --- a/weed/filer/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -86,7 +86,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = weed_util.MaybeGzipData(value) } diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 966686ed9..d68493bd7 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -88,7 +88,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = weed_util.MaybeGzipData(value) } diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index e448f0093..d21515bd4 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -177,7 +177,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = weed_util.MaybeGzipData(value) } diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index c12354ad6..83686bfe7 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -107,7 +107,7 @@ func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index 0cdf58d7f..89684647b 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -40,7 +40,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer. return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index deccf8922..7a34092a0 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -52,7 +52,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } diff --git a/weed/filer/redis3/universal_redis_store.go b/weed/filer/redis3/universal_redis_store.go index f04ee493d..10a87e2a4 100644 --- a/weed/filer/redis3/universal_redis_store.go +++ b/weed/filer/redis3/universal_redis_store.go @@ -40,7 +40,7 @@ func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } diff --git a/weed/filer/redis_lua/universal_redis_store.go b/weed/filer/redis_lua/universal_redis_store.go index 9674ac03f..0ab0f2f24 100644 --- a/weed/filer/redis_lua/universal_redis_store.go +++ b/weed/filer/redis_lua/universal_redis_store.go @@ -53,7 +53,7 @@ func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *fil return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) } - if len(entry.Chunks) > 50 { + if len(entry.Chunks) > filer.CountEntryChunksForGzip { value = util.MaybeGzipData(value) } diff --git a/weed/filer/ydb/readme.md b/weed/filer/ydb/readme.md index 90d7a18e9..2221e13b5 100644 --- a/weed/filer/ydb/readme.md +++ b/weed/filer/ydb/readme.md @@ -8,14 +8,9 @@ options: ``` [ydb] enabled=true -db_name="seaweedfs" -servers=["http://localhost:8529"] -#basic auth -user="root" -pass="test" - -# tls settings -insecure_skip_verify=true +prefix="seaweedfs" +useBucketPrefix=true +coonectionUrl=grpcs://ydb-ru.yandex.net:2135/?database=/ru/home/username/db ``` get ydb types diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index 57b282a7a..fdfc8bcb1 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -67,6 +67,6 @@ const ( SELECT name, meta FROM file_meta - WHERE dir_hash == $dir_hash AND directory == $directory and name %v $start_name and name LIKE '$prefix%' + WHERE dir_hash == $dir_hash AND directory == $directory and name %s $start_name and name LIKE '$prefix%%' ORDER BY name ASC LIMIT $limit;` ) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index aedc11ec5..d82fc4da7 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -15,6 +15,10 @@ import ( "time" ) +const ( + defaultConnectionTimeOut = 10 +) + var ( roTX = table.TxControl( table.BeginTx(table.WithOnlineReadOnly()), @@ -29,8 +33,6 @@ var ( type YdbStore struct { SupportBucketTable bool DB *connect.Connection - connParams connect.ConnectParams - connCtx context.Context dirBuckets string tablePathPrefix string } @@ -44,16 +46,27 @@ func (store *YdbStore) GetName() string { } func (store *YdbStore) Initialize(configuration util.Configuration, prefix string) (err error) { - return store.initialize(configuration.GetString(prefix + "coonectionUrl")) + return store.initialize( + configuration.GetString("filer.options.buckets_folder"), + configuration.GetString(prefix+"coonectionUrl"), + configuration.GetString(prefix+"tablePathPrefix"), + configuration.GetBool(prefix+"useBucketPrefix"), + configuration.GetInt(prefix+"connectionTimeOut"), + ) } -func (store *YdbStore) initialize(sqlUrl string) (err error) { - store.SupportBucketTable = false +func (store *YdbStore) initialize(dirBuckets string, sqlUrl string, tablePathPrefix string, useBucketPrefix bool, connectionTimeOut int) (err error) { + store.dirBuckets = dirBuckets + store.tablePathPrefix = tablePathPrefix + store.SupportBucketTable = useBucketPrefix + if connectionTimeOut == 0 { + connectionTimeOut = defaultConnectionTimeOut + } var cancel context.CancelFunc - store.connCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + connCtx, cancel := context.WithTimeout(context.Background(), time.Duration(connectionTimeOut)*time.Second) defer cancel() - store.connParams = connect.MustConnectionString(sqlUrl) - store.DB, err = connect.New(store.connCtx, store.connParams) + connParams := connect.MustConnectionString(sqlUrl) + store.DB, err = connect.New(connCtx, connParams) if err != nil { store.DB.Close() store.DB = nil @@ -61,7 +74,7 @@ func (store *YdbStore) initialize(sqlUrl string) (err error) { } defer store.DB.Close() - if err = store.DB.EnsurePathExists(store.connCtx, store.connParams.Database()); err != nil { + if err = store.DB.EnsurePathExists(connCtx, connParams.Database()); err != nil { return fmt.Errorf("connect to %s error:%v", sqlUrl, err) } return nil @@ -73,6 +86,11 @@ func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Ent if err != nil { return fmt.Errorf("encode %s: %s", entry.FullPath, err) } + + if len(entry.Chunks) > filer.CountEntryChunksForGzip { + meta = util.MaybeGzipData(meta) + } + fileMeta := FileMeta{util.HashStringToLong(dir), name, dir, meta} return table.Retry(ctx, store.DB.Table().Pool(), table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { @@ -114,7 +132,7 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e } defer res.Close() - for res.NextSet() { + for res.NextResultSet(ctx) { for res.NextRow() { res.SeekItem("meta") entry.FullPath = fullpath @@ -251,17 +269,21 @@ func (store *YdbStore) Shutdown() { } func (store *YdbStore) getPrefix(dir string) string { + if !store.SupportBucketTable { + return store.tablePathPrefix + } + prefixBuckets := store.dirBuckets + "/" if strings.HasPrefix(dir, prefixBuckets) { // detect bucket bucketAndDir := dir[len(prefixBuckets):] if t := strings.Index(bucketAndDir, "/"); t > 0 { - return bucketAndDir[:t] + return path.Join(bucketAndDir[:t], store.tablePathPrefix) } } - return "" + return store.tablePathPrefix } func (store *YdbStore) withPragma(prefix, query string) string { - return `PRAGMA TablePathPrefix("` + path.Join(store.tablePathPrefix, prefix) + `");` + query + return `PRAGMA TablePathPrefix("` + prefix + `");` + query } From 50c4f62ed4c6b4cd92893a4d23e960370975691a Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 2 May 2022 02:07:47 +0500 Subject: [PATCH 05/18] ydb-go-sdk move to v3 --- go.mod | 12 +- go.sum | 19 ++ weed/command/imports.go | 1 + weed/command/scaffold/filer.toml | 11 ++ weed/filer/ydb/readme.md | 16 +- weed/filer/ydb/ydb_store.go | 269 +++++++++++++++-------------- weed/filer/ydb/ydb_store_kv.go | 104 +++++------ weed/filer/ydb/ydb_types.go | 13 ++ weed/filer/ydb/ydb_types_ydbgen.go | 194 --------------------- weed/server/filer_server.go | 1 + 10 files changed, 257 insertions(+), 383 deletions(-) delete mode 100644 weed/filer/ydb/ydb_types_ydbgen.go diff --git a/go.mod b/go.mod index 5dbd0c6d7..9f5d54352 100644 --- a/go.mod +++ b/go.mod @@ -124,15 +124,15 @@ require ( golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd golang.org/x/image v0.0.0-20200119044424-58c23975cae1 - golang.org/x/net v0.0.0-20220412020605-290c469a71a5 + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.0.0-20220412211240-33da011f77ad + golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/api v0.75.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect + google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e // indirect google.golang.org/grpc v1.46.0 google.golang.org/protobuf v1.28.0 gopkg.in/inf.v0 v0.9.1 // indirect @@ -203,8 +203,12 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/tinylib/msgp v1.1.6 // indirect - github.com/yandex-cloud/ydb-go-sdk/v2 v2.12.0 // indirect + github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect + github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.0 // indirect + github.com/ydb-platform/ydb-go-sdk/v3 v3.24.2 // indirect + github.com/ydb-platform/ydb-go-yc v0.6.1 // indirect + github.com/ydb-platform/ydb-go-yc-metadata v0.0.9 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect go.uber.org/atomic v1.9.0 // indirect diff --git a/go.sum b/go.sum index 4f0276f94..8d50bc860 100644 --- a/go.sum +++ b/go.sum @@ -937,11 +937,23 @@ github.com/xdg-go/scram v1.1.0 h1:d70R37I0HrDLsafRrMBXyrD4lmQbCHE873t00Vr0gm0= github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/yandex-cloud/go-genproto v0.0.0-20210809082946-a97da516c588 h1:Lbz8X5Nre0Lg5QgCblmo0AhScWxeN3CVnX+mZ5Hxksk= github.com/yandex-cloud/go-genproto v0.0.0-20210809082946-a97da516c588/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= +github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8= +github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= github.com/yandex-cloud/ydb-go-sdk/v2 v2.12.0 h1:TjMLCV3Poata+0Nw+Oa/ztEFagFrochAkGg91PG49jk= github.com/yandex-cloud/ydb-go-sdk/v2 v2.12.0/go.mod h1:L09Rymwmcpp9QMCbfGAlMtJ4kLFFsbGjkP3qv58r2yY= github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc h1:xvTP0fhYNm+Ws+xC34jzF9EdorPUKkucJr0TyybqVSk= github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.0 h1:74zGbvLn5kwLkkVoicJWzmvWhaIGdIWLUr1tfaMul08= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.0/go.mod h1:9E5eeVy08G/cu5azQVYGHduqxa6hz/dyZzJDjDTE010= +github.com/ydb-platform/ydb-go-sdk/v3 v3.9.0/go.mod h1:/KCidORSzHOsn+j46Iqb8Tf6UXNhPWRMcrMieh+i9Xw= +github.com/ydb-platform/ydb-go-sdk/v3 v3.24.2 h1:hFwtj5icsW5s3+sDXkmEa+rZN6ez9f4FeIOv2AYBrMk= +github.com/ydb-platform/ydb-go-sdk/v3 v3.24.2/go.mod h1:/KCidORSzHOsn+j46Iqb8Tf6UXNhPWRMcrMieh+i9Xw= +github.com/ydb-platform/ydb-go-yc v0.6.1 h1:DBw32JwTOsfFGnMlwri2f7C2joYvFnLNYEQAopID/Qg= +github.com/ydb-platform/ydb-go-yc v0.6.1/go.mod h1:iMalotfQEHibqPDNkwn0oT2UC5ieS5j6teIuGgPzaSE= +github.com/ydb-platform/ydb-go-yc-metadata v0.0.9 h1:jymJK3FVUphDa5q6oyjLODXLc34AR+aFTMiybacMLy0= +github.com/ydb-platform/ydb-go-yc-metadata v0.0.9/go.mod h1:L7zqxXrf3DXY2CWd9T9JBiPVpdooGBxJr4CPtWiMLCg= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1129,6 +1141,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1263,6 +1277,8 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 h1:Js08h5hqB5xyWR789+QqueR6sDE8mk+YvpETZ+F6X9Y= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1504,6 +1520,8 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 h1:myaecH64R0bIEDjNORIel4iXubqzaHU1K2z8ajBwWcM= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e h1:gMjH4zLGs9m+dGzR7qHCHaXMOwsJHJKKkHtyXhtOrJk= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1530,6 +1548,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= diff --git a/weed/command/imports.go b/weed/command/imports.go index 5b3195907..04079b162 100644 --- a/weed/command/imports.go +++ b/weed/command/imports.go @@ -32,4 +32,5 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/redis2" _ "github.com/chrislusf/seaweedfs/weed/filer/redis3" _ "github.com/chrislusf/seaweedfs/weed/filer/sqlite" + _ "github.com/chrislusf/seaweedfs/weed/filer/ydb" ) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 0a505bbdc..3a920d80b 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -295,6 +295,17 @@ password="" # skip tls cert validation insecure_skip_verify = true +[ydb] +enabled = false +useBucketPrefix=true +dsn="grpc://localhost:2136?database=/local" +prefix="en" +poolSizeLimit=50 +# Authenticate produced with one of next environment variables: +# YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS= — used service account key file by path +# YDB_ANONYMOUS_CREDENTIALS="1" — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation +# YDB_METADATA_CREDENTIALS="1" — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function +# YDB_ACCESS_TOKEN_CREDENTIALS= — used for authenticate to YDB with short-life access token. For example, access token may be IAM token ########################## ########################## diff --git a/weed/filer/ydb/readme.md b/weed/filer/ydb/readme.md index 2221e13b5..fd6c5c0a2 100644 --- a/weed/filer/ydb/readme.md +++ b/weed/filer/ydb/readme.md @@ -1,6 +1,7 @@ ## YDB database: https://github.com/ydb-platform/ydb + go driver: https://github.com/ydb-platform/ydb-go-sdk options: @@ -8,15 +9,18 @@ options: ``` [ydb] enabled=true +dsn=grpcs://ydb-ru.yandex.net:2135/?database=/ru/home/username/db prefix="seaweedfs" useBucketPrefix=true -coonectionUrl=grpcs://ydb-ru.yandex.net:2135/?database=/ru/home/username/db +poolSizeLimit=50 ``` -get ydb types -``` -ydbgen -dir weed/filer/ydb -``` +Authenticate produced with one of next environment variables: + * `YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=` — used service account key file by path + * `YDB_ANONYMOUS_CREDENTIALS="1"` — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation + * `YDB_METADATA_CREDENTIALS="1"` — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function + * `YDB_ACCESS_TOKEN_CREDENTIALS=` — used for authenticate to YDB with short-life access token. For example, access token may be IAM token + * `YDB_CONNECTION_STRING="grpcs://endpoint/?database=database"` -i test using this dev database: + * i test using this dev database: `make dev_ydb` diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index d82fc4da7..4f3477471 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -7,9 +7,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/yandex-cloud/ydb-go-sdk/v2" - "github.com/yandex-cloud/ydb-go-sdk/v2/connect" - "github.com/yandex-cloud/ydb-go-sdk/v2/table" + environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" + ydb "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/sugar" + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" "path" "strings" "time" @@ -24,15 +27,12 @@ var ( table.BeginTx(table.WithOnlineReadOnly()), table.CommitTx(), ) - rwTX = table.TxControl( - table.BeginTx(table.WithSerializableReadWrite()), - table.CommitTx(), - ) + rwTX = table.DefaultTxControl() ) type YdbStore struct { SupportBucketTable bool - DB *connect.Connection + DB ydb.Connection dirBuckets string tablePathPrefix string } @@ -48,34 +48,43 @@ func (store *YdbStore) GetName() string { func (store *YdbStore) Initialize(configuration util.Configuration, prefix string) (err error) { return store.initialize( configuration.GetString("filer.options.buckets_folder"), - configuration.GetString(prefix+"coonectionUrl"), + configuration.GetString(prefix+"dsn"), configuration.GetString(prefix+"tablePathPrefix"), configuration.GetBool(prefix+"useBucketPrefix"), configuration.GetInt(prefix+"connectionTimeOut"), + configuration.GetInt(prefix+"poolSizeLimit"), ) } -func (store *YdbStore) initialize(dirBuckets string, sqlUrl string, tablePathPrefix string, useBucketPrefix bool, connectionTimeOut int) (err error) { +func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, connectionTimeOut int, poolSizeLimit int) (err error) { store.dirBuckets = dirBuckets store.tablePathPrefix = tablePathPrefix store.SupportBucketTable = useBucketPrefix + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() if connectionTimeOut == 0 { connectionTimeOut = defaultConnectionTimeOut } - var cancel context.CancelFunc - connCtx, cancel := context.WithTimeout(context.Background(), time.Duration(connectionTimeOut)*time.Second) - defer cancel() - connParams := connect.MustConnectionString(sqlUrl) - store.DB, err = connect.New(connCtx, connParams) - if err != nil { - store.DB.Close() - store.DB = nil - return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err) + opts := []ydb.Option{ + environ.WithEnvironCredentials(ctx), + ydb.WithDialTimeout(time.Duration(connectionTimeOut) * time.Second), } - defer store.DB.Close() - - if err = store.DB.EnsurePathExists(connCtx, connParams.Database()); err != nil { - return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + if poolSizeLimit > 0 { + opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit)) + } + store.DB, err = ydb.Open(ctx, dsn, opts...) + if err != nil { + _ = store.DB.Close(ctx) + store.DB = nil + return fmt.Errorf("can not connect to %s error:%v", dsn, err) + } + defer func() { _ = store.DB.Close(ctx) }() + store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix) + if err = sugar.RemoveRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { + return fmt.Errorf("RemoveRecursive %s : %v", store.tablePathPrefix, err) + } + if err = sugar.MakeRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { + return fmt.Errorf("MakeRecursive %s : %v", store.tablePathPrefix, err) } return nil } @@ -92,16 +101,14 @@ func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Ent } fileMeta := FileMeta{util.HashStringToLong(dir), name, dir, meta} - return table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), query)) - if err != nil { - return err - } - _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) - return err - }), - ) + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), query)) + if err != nil { + return fmt.Errorf("Prepare %s : %v", dir, err) + } + _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) + return err + }) } func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { @@ -114,68 +121,71 @@ func (store *YdbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { dir, name := fullpath.DirAndName() - var res *table.Result - err = table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), findQuery)) - if err != nil { - return err + var data []byte + entryFound := false + err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), findQuery)) + if err != nil { + return fmt.Errorf("Prepare %s : %v", entry.FullPath, err) + } + _, res, err := stmt.Execute(ctx, roTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$name", types.UTF8Value(name)))) + if err != nil { + return fmt.Errorf("Execute %s : %v", entry.FullPath, err) + } + defer func() { + _ = res.Close() + }() + for res.NextRow() { + if err := res.ScanNamed(named.Required("meta", &data)); err != nil { + return fmt.Errorf("scanNamed %s : %v", entry.FullPath, err) } - _, res, err = stmt.Execute(ctx, roTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$name", ydb.UTF8Value(name)))) - return err - }), - ) + entryFound = true + return nil + } + return res.Err() + }) if err != nil { return nil, err } - defer res.Close() - - for res.NextResultSet(ctx) { - for res.NextRow() { - res.SeekItem("meta") - entry.FullPath = fullpath - if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(res.String())); err != nil { - return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) - } - return entry, nil - } + if !entryFound { + return nil, filer_pb.ErrNotFound + } + entry.FullPath = fullpath + if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { + return nil, fmt.Errorf("decode %s : %v", entry.FullPath, err) } - return nil, filer_pb.ErrNotFound + return entry, nil } func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { dir, name := fullpath.DirAndName() - return table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteQuery)) - if err != nil { - return err - } - _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$name", ydb.UTF8Value(name)))) - return err - }), - ) + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteQuery)) + if err != nil { + return fmt.Errorf("Prepare %s : %v", dir, err) + } + _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$name", types.UTF8Value(name)))) + return err + }) } func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { dir, _ := fullpath.DirAndName() - return table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteFolderChildrenQuery)) - if err != nil { - return err - } - _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$directory", ydb.UTF8Value(dir)))) - return err - }), - ) + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteFolderChildrenQuery)) + if err != nil { + return fmt.Errorf("Prepare %s : %v", dir, err) + } + _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$directory", types.UTF8Value(dir)))) + return err + }) } func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { @@ -184,62 +194,60 @@ func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.Fu func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { dir := string(dirPath) - var res *table.Result startFileCompOp := ">" if includeStartFile { startFileCompOp = ">=" } - err = table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), fmt.Sprintf(ListDirectoryQuery, startFileCompOp))) - if err != nil { - return err - } - _, res, err = stmt.Execute(ctx, roTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", ydb.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$directory", ydb.UTF8Value(dir)), - table.ValueParam("$start_name", ydb.UTF8Value(startFileName)), - table.ValueParam("$prefix", ydb.UTF8Value(prefix)), - table.ValueParam("$limit", ydb.Int64Value(limit)), - )) - return err - }), - ) - if err != nil { - return lastFileName, err - } - defer res.Close() - - for res.NextSet() { - for res.NextRow() { - res.SeekItem("name") - name := res.UTF8() - res.SeekItem("meta") - data := res.String() - if res.Err() != nil { - glog.V(0).Infof("scan %s : %v", dirPath, err) - return lastFileName, fmt.Errorf("scan %s: %v", dirPath, err) - } - lastFileName = name - - entry := &filer.Entry{ - FullPath: util.NewFullPath(dir, name), - } - if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { - glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) - return lastFileName, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) - } - - if !eachEntryFunc(entry) { - break + err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), fmt.Sprintf(ListDirectoryQuery, startFileCompOp))) + if err != nil { + return fmt.Errorf("Prepare %s : %v", dir, err) + } + _, res, err := stmt.Execute(ctx, roTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$directory", types.UTF8Value(dir)), + table.ValueParam("$start_name", types.UTF8Value(startFileName)), + table.ValueParam("$prefix", types.UTF8Value(prefix)), + table.ValueParam("$limit", types.Int64Value(limit)), + )) + if err != nil { + return fmt.Errorf("Execute %s : %v", dir, err) + } + defer func() { + _ = res.Close() + }() + for res.NextResultSet(ctx) { + for res.NextRow() { + var name string + var data []byte + if err := res.ScanNamed( + named.Required("name", &name), + named.Required("meta", &data)); err != nil { + return fmt.Errorf("scanNamed %s : %v", dir, err) + } + lastFileName = name + entry := &filer.Entry{ + FullPath: util.NewFullPath(dir, name), + } + if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { + glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) + return fmt.Errorf("scan decode %s : %v", entry.FullPath, err) + } + if !eachEntryFunc(entry) { + break + } } } + return res.Err() + }) + if err != nil { + return lastFileName, err } return lastFileName, nil } func (store *YdbStore) BeginTransaction(ctx context.Context) (context.Context, error) { - session, err := store.DB.Table().Pool().Create(ctx) + session, err := store.DB.Table().CreateSession(ctx) if err != nil { return ctx, err } @@ -251,21 +259,22 @@ func (store *YdbStore) BeginTransaction(ctx context.Context) (context.Context, e } func (store *YdbStore) CommitTransaction(ctx context.Context) error { - if tx, ok := ctx.Value("tx").(*table.Transaction); ok { - return tx.Commit(ctx) + if tx, ok := ctx.Value("tx").(table.Transaction); ok { + _, err := tx.CommitTx(ctx) + return err } return nil } func (store *YdbStore) RollbackTransaction(ctx context.Context) error { - if tx, ok := ctx.Value("tx").(*table.Transaction); ok { + if tx, ok := ctx.Value("tx").(table.Transaction); ok { return tx.Rollback(ctx) } return nil } func (store *YdbStore) Shutdown() { - store.DB.Close() + _ = store.DB.Close(context.Background()) } func (store *YdbStore) getPrefix(dir string) string { diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index 3473d756d..b2a7af74f 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -5,68 +5,74 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" - "github.com/yandex-cloud/ydb-go-sdk/v2" - "github.com/yandex-cloud/ydb-go-sdk/v2/table" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" ) func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { dirStr, dirHash, name := abstract_sql.GenDirAndName(key) fileMeta := FileMeta{dirHash, name, dirStr, value} - return table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), insertQuery)) - if err != nil { - return fmt.Errorf("kv put: %v", err) - } - _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) - return fmt.Errorf("kv put: %v", err) - }), - ) + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), insertQuery)) + if err != nil { + return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) + } + _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) + if err != nil { + return fmt.Errorf("kv put %s: %v", util.NewFullPath(dirStr, name), err) + } + return nil + }) } func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { dirStr, dirHash, name := abstract_sql.GenDirAndName(key) - var res *table.Result - err = table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), findQuery)) - if err != nil { - return err - } - _, res, err = stmt.Execute(ctx, roTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", ydb.Int64Value(dirHash)), - table.ValueParam("$name", ydb.UTF8Value(name)))) - return err - }), - ) - if err != nil { - return nil, fmt.Errorf("kv get: %v", err) - } - defer res.Close() - - for res.NextResultSet(ctx) { - for res.NextRow() { - if err = res.Scan(&value); err != nil { - return nil, fmt.Errorf("kv get: %v", err) - } - return + valueFound := false + err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), findQuery)) + if err != nil { + return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) } + _, res, err := stmt.Execute(ctx, roTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(dirHash)), + table.ValueParam("$name", types.UTF8Value(name)))) + if err != nil { + return fmt.Errorf("kv get %s: %v", util.NewFullPath(dirStr, name), err) + } + defer func() { _ = res.Close() }() + for res.NextRow() { + if err := res.ScanNamed(named.Required("meta", &value)); err != nil { + return fmt.Errorf("scanNamed %s : %v", util.NewFullPath(dirStr, name), err) + } + valueFound = true + return nil + } + return res.Err() + }) + + if !valueFound { + return nil, filer.ErrKvNotFound } - return nil, filer.ErrKvNotFound + + return value, nil } func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { dirStr, dirHash, name := abstract_sql.GenDirAndName(key) - return table.Retry(ctx, store.DB.Table().Pool(), - table.OperationFunc(func(ctx context.Context, s *table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), deleteQuery)) - if err != nil { - return fmt.Errorf("kv delete: %s", err) - } - _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", ydb.Int64Value(dirHash)), - table.ValueParam("$name", ydb.UTF8Value(name)))) - return fmt.Errorf("kv delete: %s", err) - }), - ) + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), insertQuery)) + if err != nil { + return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) + } + _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(dirHash)), + table.ValueParam("$name", types.UTF8Value(name)))) + if err != nil { + return fmt.Errorf("kv delete %s: %v", util.NewFullPath(dirStr, name), err) + } + return nil + }) + } diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index 5bc0d2afd..fb1df3eee 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -1,5 +1,10 @@ package ydb +import ( + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" +) + //go:generate ydbgen //ydb:gen @@ -12,3 +17,11 @@ type FileMeta struct { //ydb:gen scan,value type FileMetas []FileMeta + +func (fm *FileMeta) QueryParameters() *table.QueryParameters { + return table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(fm.DirHash)), + table.ValueParam("$name", types.UTF8Value(fm.Name)), + table.ValueParam("$directory", types.UTF8Value(fm.Directory)), + table.ValueParam("$meta", types.StringValue(fm.Meta))) +} diff --git a/weed/filer/ydb/ydb_types_ydbgen.go b/weed/filer/ydb/ydb_types_ydbgen.go deleted file mode 100644 index 8bd1906f9..000000000 --- a/weed/filer/ydb/ydb_types_ydbgen.go +++ /dev/null @@ -1,194 +0,0 @@ -// Code generated by ydbgen; DO NOT EDIT. - -package ydb - -import ( - "strconv" - - "github.com/yandex-cloud/ydb-go-sdk/v2" - "github.com/yandex-cloud/ydb-go-sdk/v2/table" -) - -var ( - _ = strconv.Itoa - _ = ydb.StringValue - _ = table.NewQueryParameters -) - -func (f *FileMeta) Scan(res *table.Result) (err error) { - res.SeekItem("dir_hash") - f.DirHash = res.Int64() - - res.SeekItem("name") - f.Name = res.UTF8() - - res.SeekItem("directory") - f.Directory = res.UTF8() - - return res.Err() -} - -func (f *FileMeta) QueryParameters() *table.QueryParameters { - var v0 ydb.Value - { - vp0 := ydb.Int64Value(f.DirHash) - v0 = vp0 - } - var v1 ydb.Value - { - vp0 := ydb.UTF8Value(f.Name) - v1 = vp0 - } - var v2 ydb.Value - { - vp0 := ydb.UTF8Value(f.Directory) - v2 = vp0 - } - return table.NewQueryParameters( - table.ValueParam("$dir_hash", v0), - table.ValueParam("$name", v1), - table.ValueParam("$directory", v2), - ) -} - -func (f *FileMeta) StructValue() ydb.Value { - var v0 ydb.Value - { - var v1 ydb.Value - { - vp0 := ydb.Int64Value(f.DirHash) - v1 = vp0 - } - var v2 ydb.Value - { - vp0 := ydb.UTF8Value(f.Name) - v2 = vp0 - } - var v3 ydb.Value - { - vp0 := ydb.UTF8Value(f.Directory) - v3 = vp0 - } - v0 = ydb.StructValue( - ydb.StructFieldValue("dir_hash", v1), - ydb.StructFieldValue("name", v2), - ydb.StructFieldValue("directory", v3), - ) - } - return v0 -} - -func (f *FileMeta) StructType() ydb.Type { - var t0 ydb.Type - { - fs0 := make([]ydb.StructOption, 3) - var t1 ydb.Type - { - tp0 := ydb.TypeInt64 - t1 = tp0 - } - fs0[0] = ydb.StructField("dir_hash", t1) - var t2 ydb.Type - { - tp0 := ydb.TypeUTF8 - t2 = tp0 - } - fs0[1] = ydb.StructField("name", t2) - var t3 ydb.Type - { - tp0 := ydb.TypeUTF8 - t3 = tp0 - } - fs0[2] = ydb.StructField("directory", t3) - t0 = ydb.Struct(fs0...) - } - return t0 -} - -func (fs *FileMetas) Scan(res *table.Result) (err error) { - for res.NextRow() { - var x0 FileMeta - res.SeekItem("dir_hash") - x0.DirHash = res.Int64() - - res.SeekItem("name") - x0.Name = res.UTF8() - - res.SeekItem("directory") - x0.Directory = res.UTF8() - - if res.Err() == nil { - *fs = append(*fs, x0) - } - } - return res.Err() -} - -func (fs FileMetas) ListValue() ydb.Value { - var list0 ydb.Value - vs0 := make([]ydb.Value, len(fs)) - for i0, item0 := range fs { - var v0 ydb.Value - { - var v1 ydb.Value - { - var v2 ydb.Value - { - vp0 := ydb.Int64Value(item0.DirHash) - v2 = vp0 - } - var v3 ydb.Value - { - vp0 := ydb.UTF8Value(item0.Name) - v3 = vp0 - } - var v4 ydb.Value - { - vp0 := ydb.UTF8Value(item0.Directory) - v4 = vp0 - } - v1 = ydb.StructValue( - ydb.StructFieldValue("dir_hash", v2), - ydb.StructFieldValue("name", v3), - ydb.StructFieldValue("directory", v4), - ) - } - v0 = v1 - } - vs0[i0] = v0 - } - if len(vs0) == 0 { - var t1 ydb.Type - { - var t2 ydb.Type - { - fs0 := make([]ydb.StructOption, 3) - var t3 ydb.Type - { - tp0 := ydb.TypeInt64 - t3 = tp0 - } - fs0[0] = ydb.StructField("dir_hash", t3) - var t4 ydb.Type - { - tp0 := ydb.TypeUTF8 - t4 = tp0 - } - fs0[1] = ydb.StructField("name", t4) - var t5 ydb.Type - { - tp0 := ydb.TypeUTF8 - t5 = tp0 - } - fs0[2] = ydb.StructField("directory", t5) - t2 = ydb.Struct(fs0...) - } - t1 = t2 - } - t0 := ydb.List(t1) - list0 = ydb.ZeroValue(t0) - } else { - list0 = ydb.ListValue(vs0...) - } - return list0 -} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 8779e9ac0..928aad253 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -38,6 +38,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer/redis2" _ "github.com/chrislusf/seaweedfs/weed/filer/redis3" _ "github.com/chrislusf/seaweedfs/weed/filer/sqlite" + _ "github.com/chrislusf/seaweedfs/weed/filer/ydb" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/notification" _ "github.com/chrislusf/seaweedfs/weed/notification/aws_sqs" From a3e48831f396b9f4a1ded9c35ef2e6474ffd618c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 2 May 2022 12:42:20 +0500 Subject: [PATCH 06/18] ydb BucketAware interface --- weed/filer/ydb/ydb_queries.go | 24 +++---- weed/filer/ydb/ydb_store.go | 127 ++++++++++++++++++++++++++++----- weed/filer/ydb/ydb_store_kv.go | 8 +-- weed/filer/ydb/ydb_types.go | 16 ++++- 4 files changed, 136 insertions(+), 39 deletions(-) diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index fdfc8bcb1..6e0939948 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -1,23 +1,15 @@ package ydb -const ( - createQuery = ` - PRAGMA TablePathPrefix("%s"); - CREATE TABLE file_meta ( - dir_hash int64, - name Utf8, - directory Utf8, - meta String, - PRIMARY KEY (dir_hash, name) - );` +import asql "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" +const ( insertQuery = ` DECLARE $dir_hash int64; DECLARE $name AS Utf8; DECLARE $directory AS Utf8; DECLARE $meta AS String; - UPSERT INTO file_meta + UPSERT INTO ` + asql.DEFAULT_TABLE + ` (dir_hash, name, directory, meta) VALUES ($dir_hash, $name, $directory, $meta);` @@ -28,7 +20,7 @@ const ( DECLARE $directory AS Utf8; DECLARE $meta AS String; - REPLACE INTO file_meta + REPLACE INTO ` + asql.DEFAULT_TABLE + ` (dir_hash, name, directory, meta) VALUES ($dir_hash, $name, $directory, $meta) @@ -38,7 +30,7 @@ const ( DECLARE $dir_hash int64; DECLARE $name AS Utf8; - DELETE FROM file_meta + DELETE FROM ` + asql.DEFAULT_TABLE + ` WHERE dir_hash == $dir_hash AND name == $name; COMMIT;` @@ -54,11 +46,11 @@ const ( DECLARE $dir_hash int64; DECLARE $directory AS Utf8; - DELETE FROM file_meta + DELETE FROM ` + asql.DEFAULT_TABLE + ` WHERE dir_hash == $dir_hash AND directory == $directory; COMMIT;` - ListDirectoryQuery = ` + listDirectoryQuery = ` DECLARE $dir_hash int64; DECLARE $directory AS Utf8; DECLARE $start_name AS Utf8; @@ -66,7 +58,7 @@ const ( DECLARE $limit AS int64; SELECT name, meta - FROM file_meta + FROM ` + asql.DEFAULT_TABLE + ` WHERE dir_hash == $dir_hash AND directory == $directory and name %s $start_name and name LIKE '$prefix%%' ORDER BY name ASC LIMIT $limit;` ) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 4f3477471..9f136703b 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -4,17 +4,20 @@ import ( "context" "fmt" "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" - environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" - ydb "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk-auth-environ" + "github.com/ydb-platform/ydb-go-sdk/v3" "github.com/ydb-platform/ydb-go-sdk/v3/sugar" "github.com/ydb-platform/ydb-go-sdk/v3/table" "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" "github.com/ydb-platform/ydb-go-sdk/v3/table/types" + "os" "path" "strings" + "sync" "time" ) @@ -31,10 +34,12 @@ var ( ) type YdbStore struct { - SupportBucketTable bool DB ydb.Connection dirBuckets string tablePathPrefix string + SupportBucketTable bool + dbs map[string]bool + dbsLock sync.Mutex } func init() { @@ -60,6 +65,7 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix store.dirBuckets = dirBuckets store.tablePathPrefix = tablePathPrefix store.SupportBucketTable = useBucketPrefix + store.dbs = make(map[string]bool) ctx, cancel := context.WithCancel(context.Background()) defer cancel() if connectionTimeOut == 0 { @@ -72,6 +78,9 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix if poolSizeLimit > 0 { opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit)) } + if dsn == "" { + dsn = os.Getenv("YDB_CONNECTION_STRING") + } store.DB, err = ydb.Open(ctx, dsn, opts...) if err != nil { _ = store.DB.Close(ctx) @@ -79,6 +88,7 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix return fmt.Errorf("can not connect to %s error:%v", dsn, err) } defer func() { _ = store.DB.Close(ctx) }() + store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix) if err = sugar.RemoveRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { return fmt.Errorf("RemoveRecursive %s : %v", store.tablePathPrefix, err) @@ -86,6 +96,18 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix if err = sugar.MakeRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { return fmt.Errorf("MakeRecursive %s : %v", store.tablePathPrefix, err) } + + whoAmI, err := store.DB.Discovery().WhoAmI(ctx) + if err != nil { + return fmt.Errorf("connect to %s error:%v", dsn, err) + } + glog.V(0).Infof("connected to ydb: %s", whoAmI.String()) + + tablePath := path.Join(store.tablePathPrefix, abstract_sql.DEFAULT_TABLE) + if err := store.createTable(ctx, tablePath); err != nil { + glog.Errorf("createTable %s: %v", tablePath, err) + } + return nil } @@ -102,11 +124,11 @@ func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Ent fileMeta := FileMeta{util.HashStringToLong(dir), name, dir, meta} return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), query)) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), query)) if err != nil { return fmt.Errorf("Prepare %s : %v", dir, err) } - _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) + _, _, err = stmt.Execute(ctx, rwTX, fileMeta.queryParameters()) return err }) } @@ -124,7 +146,7 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e var data []byte entryFound := false err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), findQuery)) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), findQuery)) if err != nil { return fmt.Errorf("Prepare %s : %v", entry.FullPath, err) } @@ -163,7 +185,7 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { dir, name := fullpath.DirAndName() return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteQuery)) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), deleteQuery)) if err != nil { return fmt.Errorf("Prepare %s : %v", dir, err) } @@ -177,7 +199,7 @@ func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { dir, _ := fullpath.DirAndName() return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), deleteFolderChildrenQuery)) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), deleteFolderChildrenQuery)) if err != nil { return fmt.Errorf("Prepare %s : %v", dir, err) } @@ -199,7 +221,7 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath startFileCompOp = ">=" } err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dir), fmt.Sprintf(ListDirectoryQuery, startFileCompOp))) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), fmt.Sprintf(listDirectoryQuery, startFileCompOp))) if err != nil { return fmt.Errorf("Prepare %s : %v", dir, err) } @@ -277,22 +299,91 @@ func (store *YdbStore) Shutdown() { _ = store.DB.Close(context.Background()) } -func (store *YdbStore) getPrefix(dir string) string { +func (store *YdbStore) CanDropWholeBucket() bool { + return store.SupportBucketTable +} + +func (store *YdbStore) OnBucketCreation(bucket string) { + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + if err := store.createTable(context.Background(), bucket); err != nil { + glog.Errorf("createTable %s: %v", bucket, err) + } + + if store.dbs == nil { + return + } + store.dbs[bucket] = true +} + +func (store *YdbStore) OnBucketDeletion(bucket string) { + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + if err := store.deleteTable(context.Background(), bucket); err != nil { + glog.Errorf("deleteTable %s: %v", bucket, err) + } + + if store.dbs == nil { + return + } + delete(store.dbs, bucket) +} + +func (store *YdbStore) createTable(ctx context.Context, prefix string) error { + e, err := store.DB.Scheme().DescribePath(ctx, prefix) + if err != nil { + return fmt.Errorf("describe path %s error:%v", prefix, err) + } + if e.IsTable() { + return nil + } + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + return s.CreateTable(ctx, prefix, createTableOptions()...) + }) +} + +func (store *YdbStore) deleteTable(ctx context.Context, prefix string) error { if !store.SupportBucketTable { - return store.tablePathPrefix + return nil + } + return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + return s.DropTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE)) + }) +} + +func (store *YdbStore) getPrefix(ctx context.Context, dir string) (tablePathPrefix string) { + tablePathPrefix = store.tablePathPrefix + if !store.SupportBucketTable { + return } prefixBuckets := store.dirBuckets + "/" if strings.HasPrefix(dir, prefixBuckets) { // detect bucket bucketAndDir := dir[len(prefixBuckets):] - if t := strings.Index(bucketAndDir, "/"); t > 0 { - return path.Join(bucketAndDir[:t], store.tablePathPrefix) + t := strings.Index(bucketAndDir, "/") + if t < 0 { + return } - } - return store.tablePathPrefix -} + bucket := bucketAndDir[:t] -func (store *YdbStore) withPragma(prefix, query string) string { - return `PRAGMA TablePathPrefix("` + prefix + `");` + query + if bucket != "" { + return + } + store.dbsLock.Lock() + defer store.dbsLock.Unlock() + + if _, found := store.dbs[bucket]; !found { + if err := store.createTable(ctx, + path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)); err == nil { + store.dbs[bucket] = true + } else { + glog.Errorf("createTable %s: %v", bucket, err) + } + } + tablePathPrefix = path.Join(store.tablePathPrefix, bucket) + } + return } diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index b2a7af74f..81ac6446b 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -15,11 +15,11 @@ func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err dirStr, dirHash, name := abstract_sql.GenDirAndName(key) fileMeta := FileMeta{dirHash, name, dirStr, value} return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), insertQuery)) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), insertQuery)) if err != nil { return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) } - _, _, err = stmt.Execute(ctx, rwTX, fileMeta.QueryParameters()) + _, _, err = stmt.Execute(ctx, rwTX, fileMeta.queryParameters()) if err != nil { return fmt.Errorf("kv put %s: %v", util.NewFullPath(dirStr, name), err) } @@ -31,7 +31,7 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err dirStr, dirHash, name := abstract_sql.GenDirAndName(key) valueFound := false err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), findQuery)) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), findQuery)) if err != nil { return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) } @@ -62,7 +62,7 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { dirStr, dirHash, name := abstract_sql.GenDirAndName(key) return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, store.withPragma(store.getPrefix(dirStr), insertQuery)) + stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), insertQuery)) if err != nil { return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) } diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index fb1df3eee..3c381797a 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -2,6 +2,7 @@ package ydb import ( "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/options" "github.com/ydb-platform/ydb-go-sdk/v3/table/types" ) @@ -18,10 +19,23 @@ type FileMeta struct { //ydb:gen scan,value type FileMetas []FileMeta -func (fm *FileMeta) QueryParameters() *table.QueryParameters { +func (fm *FileMeta) queryParameters() *table.QueryParameters { return table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(fm.DirHash)), table.ValueParam("$name", types.UTF8Value(fm.Name)), table.ValueParam("$directory", types.UTF8Value(fm.Directory)), table.ValueParam("$meta", types.StringValue(fm.Meta))) } + +func createTableOptions() []options.CreateTableOption { + return []options.CreateTableOption{ + options.WithColumn("dir_hash", types.TypeUint64), + options.WithColumn("name", types.TypeUTF8), + options.WithColumn("directory", types.TypeUTF8), + options.WithColumn("meta", types.TypeString), + options.WithPrimaryKeyColumn("dir_hash", "name"), + } +} +func withPragma(prefix, query string) string { + return `PRAGMA TablePathPrefix("` + prefix + `");` + query +} From 0dc44dda63515d690161f3f60504733489189c66 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 2 May 2022 15:33:29 +0500 Subject: [PATCH 07/18] ydb do Tx or DB --- weed/filer/ydb/ydb_store.go | 114 ++++++++++++++++++------------------ weed/filer/ydb/ydb_types.go | 2 +- 2 files changed, 59 insertions(+), 57 deletions(-) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 9f136703b..8271c6e18 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -12,6 +12,7 @@ import ( "github.com/ydb-platform/ydb-go-sdk/v3" "github.com/ydb-platform/ydb-go-sdk/v3/sugar" "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/result" "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" "github.com/ydb-platform/ydb-go-sdk/v3/table/types" "os" @@ -111,6 +112,34 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix return nil } +func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) { + var res result.Result + if tx, ok := ctx.Value("tx").(table.Transaction); ok { + res, err = tx.Execute(ctx, *query, params) + if err != nil { + return fmt.Errorf("execute transaction: %v", err) + } + } else { + err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + stmt, err := s.Prepare(ctx, *query) + if err != nil { + return fmt.Errorf("prepare: %v", err) + } + _, res, err = stmt.Execute(ctx, tc, params) + if err != nil { + return fmt.Errorf("execute statement: %v", err) + } + return nil + }) + } + if err != nil && processResultFunc != nil && res != nil { + if err = processResultFunc(res); err != nil { + return fmt.Errorf("process resul: %v", err) + } + } + return err +} + func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Entry, query string) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -121,16 +150,9 @@ func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Ent if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } - + queryWithPragma := withPragma(store.getPrefix(ctx, dir), query) fileMeta := FileMeta{util.HashStringToLong(dir), name, dir, meta} - return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), query)) - if err != nil { - return fmt.Errorf("Prepare %s : %v", dir, err) - } - _, _, err = stmt.Execute(ctx, rwTX, fileMeta.queryParameters()) - return err - }) + return store.doTxOrDB(ctx, &queryWithPragma, fileMeta.queryParameters(), rwTX, nil) } func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { @@ -145,17 +167,12 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e dir, name := fullpath.DirAndName() var data []byte entryFound := false - err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), findQuery)) - if err != nil { - return fmt.Errorf("Prepare %s : %v", entry.FullPath, err) - } - _, res, err := stmt.Execute(ctx, roTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$name", types.UTF8Value(name)))) - if err != nil { - return fmt.Errorf("Execute %s : %v", entry.FullPath, err) - } + queryWithPragma := withPragma(store.getPrefix(ctx, dir), findQuery) + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$name", types.UTF8Value(name))) + + err = store.doTxOrDB(ctx, &queryWithPragma, queryParams, roTX, func(res result.Result) error { defer func() { _ = res.Close() }() @@ -184,30 +201,22 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { dir, name := fullpath.DirAndName() - return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), deleteQuery)) - if err != nil { - return fmt.Errorf("Prepare %s : %v", dir, err) - } - _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$name", types.UTF8Value(name)))) - return err - }) + queryWithPragma := withPragma(store.getPrefix(ctx, dir), deleteQuery) + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$name", types.UTF8Value(name))) + + return store.doTxOrDB(ctx, &queryWithPragma, queryParams, rwTX, nil) } func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { dir, _ := fullpath.DirAndName() - return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), deleteFolderChildrenQuery)) - if err != nil { - return fmt.Errorf("Prepare %s : %v", dir, err) - } - _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$directory", types.UTF8Value(dir)))) - return err - }) + queryWithPragma := withPragma(store.getPrefix(ctx, dir), deleteFolderChildrenQuery) + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$directory", types.UTF8Value(dir))) + + return store.doTxOrDB(ctx, &queryWithPragma, queryParams, rwTX, nil) } func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { @@ -220,21 +229,15 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath if includeStartFile { startFileCompOp = ">=" } - err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dir), fmt.Sprintf(listDirectoryQuery, startFileCompOp))) - if err != nil { - return fmt.Errorf("Prepare %s : %v", dir, err) - } - _, res, err := stmt.Execute(ctx, roTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$directory", types.UTF8Value(dir)), - table.ValueParam("$start_name", types.UTF8Value(startFileName)), - table.ValueParam("$prefix", types.UTF8Value(prefix)), - table.ValueParam("$limit", types.Int64Value(limit)), - )) - if err != nil { - return fmt.Errorf("Execute %s : %v", dir, err) - } + queryWithPragma := withPragma(store.getPrefix(ctx, dir), fmt.Sprintf(listDirectoryQuery, startFileCompOp)) + queryParams := table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$directory", types.UTF8Value(dir)), + table.ValueParam("$start_name", types.UTF8Value(startFileName)), + table.ValueParam("$prefix", types.UTF8Value(prefix)), + table.ValueParam("$limit", types.Int64Value(limit)), + ) + err = store.doTxOrDB(ctx, &queryWithPragma, queryParams, roTX, func(res result.Result) error { defer func() { _ = res.Close() }() @@ -252,7 +255,6 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath FullPath: util.NewFullPath(dir, name), } if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { - glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) return fmt.Errorf("scan decode %s : %v", entry.FullPath, err) } if !eachEntryFunc(entry) { diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index 3c381797a..8c24c4738 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -36,6 +36,6 @@ func createTableOptions() []options.CreateTableOption { options.WithPrimaryKeyColumn("dir_hash", "name"), } } -func withPragma(prefix, query string) string { +func withPragma(prefix string, query string) string { return `PRAGMA TablePathPrefix("` + prefix + `");` + query } From 319d300d48ab4689cfd821429f8247a946ffef68 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 2 May 2022 22:23:07 +0500 Subject: [PATCH 08/18] fix createDB --- docker/Makefile | 3 +++ docker/compose/local-ydb-compose.yml | 9 ++++--- weed/filer/ydb/ydb_store.go | 39 ++++++++++------------------ weed/filer/ydb/ydb_store_kv.go | 12 ++++----- weed/filer/ydb/ydb_types.go | 8 +++--- 5 files changed, 33 insertions(+), 38 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 76cdf75c8..b46dcedf1 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -49,6 +49,9 @@ dev_replicate: build dev_auditlog: build docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up +dev_ydb: build + docker-compose -f compose/local-ydb-compose.yml -p seaweedfs up + cluster: build docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up diff --git a/docker/compose/local-ydb-compose.yml b/docker/compose/local-ydb-compose.yml index ce3e7e9ed..33f550600 100644 --- a/docker/compose/local-ydb-compose.yml +++ b/docker/compose/local-ydb-compose.yml @@ -7,10 +7,8 @@ services: - 2135:2135 - 8765:8765 - 2136:2136 - volumes: - - ./seaweedfs.sql:/docker-entrypoint-initdb.d/seaweedfs.sql environment: - - YDB_DEFAULT_LOG_LEVEL=NOTICE + - YDB_DEFAULT_LOG_LEVEL=DEBUG - GRPC_TLS_PORT=2135 - GRPC_PORT=2136 - MON_PORT=8765 @@ -26,5 +24,10 @@ services: command: "server -ip=server -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" volumes: - ./master-cloud.toml:/etc/seaweedfs/master.toml + environment: + - WEED_LEVELDB2_ENABLED=false + - WEED_YDB_ENABLED=true + - WEED_YDB_DSN=grpc://ydb:2136/?database=local + - YDB_ANONYMOUS_CREDENTIALS=1 depends_on: - ydb \ No newline at end of file diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 8271c6e18..8e0815009 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -8,7 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/ydb-platform/ydb-go-sdk-auth-environ" + environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" "github.com/ydb-platform/ydb-go-sdk/v3" "github.com/ydb-platform/ydb-go-sdk/v3/sugar" "github.com/ydb-platform/ydb-go-sdk/v3/table" @@ -73,8 +73,8 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix connectionTimeOut = defaultConnectionTimeOut } opts := []ydb.Option{ - environ.WithEnvironCredentials(ctx), ydb.WithDialTimeout(time.Duration(connectionTimeOut) * time.Second), + environ.WithEnvironCredentials(ctx), } if poolSizeLimit > 0 { opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit)) @@ -84,11 +84,12 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix } store.DB, err = ydb.Open(ctx, dsn, opts...) if err != nil { - _ = store.DB.Close(ctx) - store.DB = nil - return fmt.Errorf("can not connect to %s error:%v", dsn, err) + if store.DB != nil { + _ = store.DB.Close(ctx) + store.DB = nil + } + return fmt.Errorf("can not connect to %s error: %v", dsn, err) } - defer func() { _ = store.DB.Close(ctx) }() store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix) if err = sugar.RemoveRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { @@ -98,18 +99,11 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix return fmt.Errorf("MakeRecursive %s : %v", store.tablePathPrefix, err) } - whoAmI, err := store.DB.Discovery().WhoAmI(ctx) - if err != nil { - return fmt.Errorf("connect to %s error:%v", dsn, err) - } - glog.V(0).Infof("connected to ydb: %s", whoAmI.String()) - tablePath := path.Join(store.tablePathPrefix, abstract_sql.DEFAULT_TABLE) - if err := store.createTable(ctx, tablePath); err != nil { + if err = store.createTable(ctx, tablePath); err != nil { glog.Errorf("createTable %s: %v", tablePath, err) } - - return nil + return err } func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) { @@ -309,7 +303,8 @@ func (store *YdbStore) OnBucketCreation(bucket string) { store.dbsLock.Lock() defer store.dbsLock.Unlock() - if err := store.createTable(context.Background(), bucket); err != nil { + if err := store.createTable(context.Background(), + path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)); err != nil { glog.Errorf("createTable %s: %v", bucket, err) } @@ -323,7 +318,8 @@ func (store *YdbStore) OnBucketDeletion(bucket string) { store.dbsLock.Lock() defer store.dbsLock.Unlock() - if err := store.deleteTable(context.Background(), bucket); err != nil { + if err := store.deleteTable(context.Background(), + path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)); err != nil { glog.Errorf("deleteTable %s: %v", bucket, err) } @@ -334,13 +330,6 @@ func (store *YdbStore) OnBucketDeletion(bucket string) { } func (store *YdbStore) createTable(ctx context.Context, prefix string) error { - e, err := store.DB.Scheme().DescribePath(ctx, prefix) - if err != nil { - return fmt.Errorf("describe path %s error:%v", prefix, err) - } - if e.IsTable() { - return nil - } return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { return s.CreateTable(ctx, prefix, createTableOptions()...) }) @@ -351,7 +340,7 @@ func (store *YdbStore) deleteTable(ctx context.Context, prefix string) error { return nil } return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - return s.DropTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE)) + return s.DropTable(ctx, prefix) }) } diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index 81ac6446b..d679d7246 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -17,11 +17,11 @@ func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), insertQuery)) if err != nil { - return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) + return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name).Name(), err) } _, _, err = stmt.Execute(ctx, rwTX, fileMeta.queryParameters()) if err != nil { - return fmt.Errorf("kv put %s: %v", util.NewFullPath(dirStr, name), err) + return fmt.Errorf("kv put %s: %v", util.NewFullPath(dirStr, name).Name(), err) } return nil }) @@ -39,12 +39,12 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err table.ValueParam("$dir_hash", types.Int64Value(dirHash)), table.ValueParam("$name", types.UTF8Value(name)))) if err != nil { - return fmt.Errorf("kv get %s: %v", util.NewFullPath(dirStr, name), err) + return fmt.Errorf("kv get %s: %v", util.NewFullPath(dirStr, name).Name(), err) } defer func() { _ = res.Close() }() for res.NextRow() { if err := res.ScanNamed(named.Required("meta", &value)); err != nil { - return fmt.Errorf("scanNamed %s : %v", util.NewFullPath(dirStr, name), err) + return fmt.Errorf("scanNamed %s : %v", util.NewFullPath(dirStr, name).Name(), err) } valueFound = true return nil @@ -64,13 +64,13 @@ func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), insertQuery)) if err != nil { - return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) + return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name).Name(), err) } _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(dirHash)), table.ValueParam("$name", types.UTF8Value(name)))) if err != nil { - return fmt.Errorf("kv delete %s: %v", util.NewFullPath(dirStr, name), err) + return fmt.Errorf("kv delete %s: %v", util.NewFullPath(dirStr, name).Name(), err) } return nil }) diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index 8c24c4738..4194afbc8 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -29,10 +29,10 @@ func (fm *FileMeta) queryParameters() *table.QueryParameters { func createTableOptions() []options.CreateTableOption { return []options.CreateTableOption{ - options.WithColumn("dir_hash", types.TypeUint64), - options.WithColumn("name", types.TypeUTF8), - options.WithColumn("directory", types.TypeUTF8), - options.WithColumn("meta", types.TypeString), + options.WithColumn("dir_hash", types.Optional(types.TypeUint64)), + options.WithColumn("name", types.Optional(types.TypeUTF8)), + options.WithColumn("directory", types.Optional(types.TypeUTF8)), + options.WithColumn("meta", types.Optional(types.TypeString)), options.WithPrimaryKeyColumn("dir_hash", "name"), } } From 7640e650e5e647e6d6f3c8043c1e0a7442f154c7 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 00:11:37 +0500 Subject: [PATCH 09/18] fix queries --- weed/filer/ydb/ydb_queries.go | 18 +++++++++--------- weed/filer/ydb/ydb_store.go | 4 ++-- weed/filer/ydb/ydb_store_kv.go | 8 ++++---- weed/filer/ydb/ydb_types.go | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index 6e0939948..659cc2158 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -4,7 +4,7 @@ import asql "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" const ( insertQuery = ` - DECLARE $dir_hash int64; + DECLARE $dir_hash AS int64; DECLARE $name AS Utf8; DECLARE $directory AS Utf8; DECLARE $meta AS String; @@ -15,7 +15,7 @@ const ( ($dir_hash, $name, $directory, $meta);` updateQuery = ` - DECLARE $dir_hash int64; + DECLARE $dir_hash AS int64; DECLARE $name AS Utf8; DECLARE $directory AS Utf8; DECLARE $meta AS String; @@ -27,7 +27,7 @@ const ( COMMIT;` deleteQuery = ` - DECLARE $dir_hash int64; + DECLARE $dir_hash AS int64; DECLARE $name AS Utf8; DELETE FROM ` + asql.DEFAULT_TABLE + ` @@ -35,15 +35,15 @@ const ( COMMIT;` findQuery = ` - DECLARE $dir_hash int64; + DECLARE $dir_hash AS int64; DECLARE $name AS Utf8; SELECT meta - FROM file_meta + FROM ` + asql.DEFAULT_TABLE + ` WHERE dir_hash == $dir_hash AND name == $name;` deleteFolderChildrenQuery = ` - DECLARE $dir_hash int64; + DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; DELETE FROM ` + asql.DEFAULT_TABLE + ` @@ -51,14 +51,14 @@ const ( COMMIT;` listDirectoryQuery = ` - DECLARE $dir_hash int64; + DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; DECLARE $start_name AS Utf8; DECLARE $prefix AS Utf8; - DECLARE $limit AS int64; + DECLARE $limit AS Uint64; SELECT name, meta FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash == $dir_hash AND directory == $directory and name %s $start_name and name LIKE '$prefix%%' + WHERE dir_hash == $dir_hash AND directory == $directory and name %s $start_name and name LIKE $prefix ORDER BY name ASC LIMIT $limit;` ) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 8e0815009..6d6389dde 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -228,8 +228,8 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), table.ValueParam("$directory", types.UTF8Value(dir)), table.ValueParam("$start_name", types.UTF8Value(startFileName)), - table.ValueParam("$prefix", types.UTF8Value(prefix)), - table.ValueParam("$limit", types.Int64Value(limit)), + table.ValueParam("$prefix", types.UTF8Value(prefix+"%")), + table.ValueParam("$limit", types.Uint64Value(uint64(limit))), ) err = store.doTxOrDB(ctx, &queryWithPragma, queryParams, roTX, func(res result.Result) error { defer func() { diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index d679d7246..069c35224 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -17,11 +17,11 @@ func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), insertQuery)) if err != nil { - return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name).Name(), err) + return fmt.Errorf("kv put prepare %s: %v", util.NewFullPath(dirStr, name).Name(), err) } _, _, err = stmt.Execute(ctx, rwTX, fileMeta.queryParameters()) if err != nil { - return fmt.Errorf("kv put %s: %v", util.NewFullPath(dirStr, name).Name(), err) + return fmt.Errorf("kv put execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) } return nil }) @@ -33,13 +33,13 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), findQuery)) if err != nil { - return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name), err) + return fmt.Errorf("kv get prepare %s: %v", util.NewFullPath(dirStr, name), err) } _, res, err := stmt.Execute(ctx, roTX, table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(dirHash)), table.ValueParam("$name", types.UTF8Value(name)))) if err != nil { - return fmt.Errorf("kv get %s: %v", util.NewFullPath(dirStr, name).Name(), err) + return fmt.Errorf("kv get execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) } defer func() { _ = res.Close() }() for res.NextRow() { diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index 4194afbc8..07e69e6a5 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -13,7 +13,7 @@ type FileMeta struct { DirHash int64 `ydb:"type:int64"` Name string `ydb:"type:utf8"` Directory string `ydb:"type:utf8"` - Meta []byte `ydb:"-"` + Meta []byte `ydb:"type:string"` } //ydb:gen scan,value @@ -29,7 +29,7 @@ func (fm *FileMeta) queryParameters() *table.QueryParameters { func createTableOptions() []options.CreateTableOption { return []options.CreateTableOption{ - options.WithColumn("dir_hash", types.Optional(types.TypeUint64)), + options.WithColumn("dir_hash", types.Optional(types.TypeInt64)), options.WithColumn("name", types.Optional(types.TypeUTF8)), options.WithColumn("directory", types.Optional(types.TypeUTF8)), options.WithColumn("meta", types.Optional(types.TypeString)), From f6d90e12a63687f30454ede1423c67b94fe0db57 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 00:20:39 +0500 Subject: [PATCH 10/18] fix conflicts --- go.mod | 12 +++++++++--- go.sum | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 09c393674..0ee247636 100644 --- a/go.mod +++ b/go.mod @@ -124,15 +124,15 @@ require ( golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd golang.org/x/image v0.0.0-20200119044424-58c23975cae1 - golang.org/x/net v0.0.0-20220412020605-290c469a71a5 + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 + golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/api v0.77.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.46.0 google.golang.org/protobuf v1.28.0 gopkg.in/inf.v0 v0.9.1 // indirect @@ -203,6 +203,12 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/tinylib/msgp v1.1.6 // indirect + github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect + github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.0 // indirect + github.com/ydb-platform/ydb-go-sdk/v3 v3.24.2 // indirect + github.com/ydb-platform/ydb-go-yc v0.6.1 // indirect + github.com/ydb-platform/ydb-go-yc-metadata v0.0.9 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect go.uber.org/atomic v1.9.0 // indirect diff --git a/go.sum b/go.sum index 2899f0d23..109a3e0e1 100644 --- a/go.sum +++ b/go.sum @@ -858,6 +858,19 @@ github.com/xdg-go/scram v1.1.0 h1:d70R37I0HrDLsafRrMBXyrD4lmQbCHE873t00Vr0gm0= github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8= +github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc h1:xvTP0fhYNm+Ws+xC34jzF9EdorPUKkucJr0TyybqVSk= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.0 h1:74zGbvLn5kwLkkVoicJWzmvWhaIGdIWLUr1tfaMul08= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.0/go.mod h1:9E5eeVy08G/cu5azQVYGHduqxa6hz/dyZzJDjDTE010= +github.com/ydb-platform/ydb-go-sdk/v3 v3.9.0/go.mod h1:/KCidORSzHOsn+j46Iqb8Tf6UXNhPWRMcrMieh+i9Xw= +github.com/ydb-platform/ydb-go-sdk/v3 v3.24.2 h1:hFwtj5icsW5s3+sDXkmEa+rZN6ez9f4FeIOv2AYBrMk= +github.com/ydb-platform/ydb-go-sdk/v3 v3.24.2/go.mod h1:/KCidORSzHOsn+j46Iqb8Tf6UXNhPWRMcrMieh+i9Xw= +github.com/ydb-platform/ydb-go-yc v0.6.1 h1:DBw32JwTOsfFGnMlwri2f7C2joYvFnLNYEQAopID/Qg= +github.com/ydb-platform/ydb-go-yc v0.6.1/go.mod h1:iMalotfQEHibqPDNkwn0oT2UC5ieS5j6teIuGgPzaSE= +github.com/ydb-platform/ydb-go-yc-metadata v0.0.9 h1:jymJK3FVUphDa5q6oyjLODXLc34AR+aFTMiybacMLy0= +github.com/ydb-platform/ydb-go-yc-metadata v0.0.9/go.mod h1:L7zqxXrf3DXY2CWd9T9JBiPVpdooGBxJr4CPtWiMLCg= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1037,6 +1050,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1167,6 +1182,8 @@ golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba h1:AyHWHCBVlIYI5rgEM3o+1PLd0sLPcIAoaUckGQMaWtw= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1406,6 +1423,8 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4 h1:myaecH64R0bIEDjNORIel4iXubqzaHU1K2z8ajBwWcM= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1432,6 +1451,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= From 8342f651f393a2b939de39af8a92f42077ca00fb Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 15:18:28 +0500 Subject: [PATCH 11/18] fix scanNamed --- docker/compose/local-ydb-compose.yml | 20 ++++---- weed/filer/ydb/ydb_queries.go | 12 ++--- weed/filer/ydb/ydb_store.go | 75 ++++++++++++++-------------- weed/filer/ydb/ydb_store_kv.go | 13 +++-- weed/filer/ydb/ydb_types.go | 4 +- 5 files changed, 64 insertions(+), 60 deletions(-) diff --git a/docker/compose/local-ydb-compose.yml b/docker/compose/local-ydb-compose.yml index 33f550600..a17b77b8a 100644 --- a/docker/compose/local-ydb-compose.yml +++ b/docker/compose/local-ydb-compose.yml @@ -12,7 +12,7 @@ services: - GRPC_TLS_PORT=2135 - GRPC_PORT=2136 - MON_PORT=8765 - server: + s3: image: chrislusf/seaweedfs:local ports: - 9333:9333 @@ -20,14 +20,16 @@ services: - 8084:8080 - 18084:18080 - 8888:8888 + - 8000:8000 - 18888:18888 - command: "server -ip=server -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" + command: "server -ip=s3 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false" volumes: - - ./master-cloud.toml:/etc/seaweedfs/master.toml + - ./s3.json:/etc/seaweedfs/s3.json environment: - - WEED_LEVELDB2_ENABLED=false - - WEED_YDB_ENABLED=true - - WEED_YDB_DSN=grpc://ydb:2136/?database=local - - YDB_ANONYMOUS_CREDENTIALS=1 - depends_on: - - ydb \ No newline at end of file + WEED_LEVELDB2_ENABLED: "false" + WEED_YDB_ENABLED: "true" + WEED_YDB_DSN: "grpc://ydb:2136/?database=local" + WEED_YDB_PREFIX: "seaweedfs" + YDB_ANONYMOUS_CREDENTIALS: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 \ No newline at end of file diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index 659cc2158..bc2f37b10 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -5,8 +5,8 @@ import asql "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" const ( insertQuery = ` DECLARE $dir_hash AS int64; - DECLARE $name AS Utf8; DECLARE $directory AS Utf8; + DECLARE $name AS Utf8; DECLARE $meta AS String; UPSERT INTO ` + asql.DEFAULT_TABLE + ` @@ -16,8 +16,8 @@ const ( updateQuery = ` DECLARE $dir_hash AS int64; - DECLARE $name AS Utf8; DECLARE $directory AS Utf8; + DECLARE $name AS Utf8; DECLARE $meta AS String; REPLACE INTO ` + asql.DEFAULT_TABLE + ` @@ -31,7 +31,7 @@ const ( DECLARE $name AS Utf8; DELETE FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash == $dir_hash AND name == $name; + WHERE dir_hash = $dir_hash AND name = $name; COMMIT;` findQuery = ` @@ -40,14 +40,14 @@ const ( SELECT meta FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash == $dir_hash AND name == $name;` + WHERE dir_hash = $dir_hash AND name = $name;` deleteFolderChildrenQuery = ` DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; DELETE FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash == $dir_hash AND directory == $directory; + WHERE dir_hash = $dir_hash AND directory = $directory; COMMIT;` listDirectoryQuery = ` @@ -59,6 +59,6 @@ const ( SELECT name, meta FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash == $dir_hash AND directory == $directory and name %s $start_name and name LIKE $prefix + WHERE dir_hash = $dir_hash AND directory = $directory and name %s $start_name and name LIKE $prefix ORDER BY name ASC LIMIT $limit;` ) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 6d6389dde..678f58143 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -55,7 +55,7 @@ func (store *YdbStore) Initialize(configuration util.Configuration, prefix strin return store.initialize( configuration.GetString("filer.options.buckets_folder"), configuration.GetString(prefix+"dsn"), - configuration.GetString(prefix+"tablePathPrefix"), + configuration.GetString(prefix+"prefix"), configuration.GetBool(prefix+"useBucketPrefix"), configuration.GetInt(prefix+"connectionTimeOut"), configuration.GetInt(prefix+"poolSizeLimit"), @@ -64,7 +64,6 @@ func (store *YdbStore) Initialize(configuration util.Configuration, prefix strin func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, connectionTimeOut int, poolSizeLimit int) (err error) { store.dirBuckets = dirBuckets - store.tablePathPrefix = tablePathPrefix store.SupportBucketTable = useBucketPrefix store.dbs = make(map[string]bool) ctx, cancel := context.WithCancel(context.Background()) @@ -83,7 +82,7 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix dsn = os.Getenv("YDB_CONNECTION_STRING") } store.DB, err = ydb.Open(ctx, dsn, opts...) - if err != nil { + if err != nil || store.DB == nil { if store.DB != nil { _ = store.DB.Close(ctx) store.DB = nil @@ -92,16 +91,12 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix } store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix) - if err = sugar.RemoveRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { - return fmt.Errorf("RemoveRecursive %s : %v", store.tablePathPrefix, err) - } if err = sugar.MakeRecursive(ctx, store.DB, store.tablePathPrefix); err != nil { return fmt.Errorf("MakeRecursive %s : %v", store.tablePathPrefix, err) } - tablePath := path.Join(store.tablePathPrefix, abstract_sql.DEFAULT_TABLE) - if err = store.createTable(ctx, tablePath); err != nil { - glog.Errorf("createTable %s: %v", tablePath, err) + if err = store.createTable(ctx, store.tablePathPrefix); err != nil { + glog.Errorf("createTable %s: %v", store.tablePathPrefix, err) } return err } @@ -126,9 +121,15 @@ func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *tabl return nil }) } - if err != nil && processResultFunc != nil && res != nil { - if err = processResultFunc(res); err != nil { - return fmt.Errorf("process resul: %v", err) + if err != nil { + return err + } + if res != nil { + defer func() { _ = res.Close() }() + if processResultFunc != nil { + if err = processResultFunc(res); err != nil { + return fmt.Errorf("process result: %v", err) + } } } return err @@ -167,15 +168,14 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e table.ValueParam("$name", types.UTF8Value(name))) err = store.doTxOrDB(ctx, &queryWithPragma, queryParams, roTX, func(res result.Result) error { - defer func() { - _ = res.Close() - }() - for res.NextRow() { - if err := res.ScanNamed(named.Required("meta", &data)); err != nil { - return fmt.Errorf("scanNamed %s : %v", entry.FullPath, err) + for res.NextResultSet(ctx) { + for res.NextRow() { + if err = res.ScanNamed(named.OptionalWithDefault("meta", &data)); err != nil { + return fmt.Errorf("scanNamed %s : %v", fullpath, err) + } + entryFound = true + return nil } - entryFound = true - return nil } return res.Err() }) @@ -185,9 +185,12 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e if !entryFound { return nil, filer_pb.ErrNotFound } - entry.FullPath = fullpath + + entry = &filer.Entry{ + FullPath: fullpath, + } if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil { - return nil, fmt.Errorf("decode %s : %v", entry.FullPath, err) + return nil, fmt.Errorf("decode %s : %v", fullpath, err) } return entry, nil @@ -232,17 +235,14 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath table.ValueParam("$limit", types.Uint64Value(uint64(limit))), ) err = store.doTxOrDB(ctx, &queryWithPragma, queryParams, roTX, func(res result.Result) error { - defer func() { - _ = res.Close() - }() + var name string + var data []byte for res.NextResultSet(ctx) { for res.NextRow() { - var name string - var data []byte if err := res.ScanNamed( - named.Required("name", &name), - named.Required("meta", &data)); err != nil { - return fmt.Errorf("scanNamed %s : %v", dir, err) + named.OptionalWithDefault("name", &name), + named.OptionalWithDefault("meta", &data)); err != nil { + return fmt.Errorf("list scanNamed %s : %v", dir, err) } lastFileName = name entry := &filer.Entry{ @@ -304,7 +304,7 @@ func (store *YdbStore) OnBucketCreation(bucket string) { defer store.dbsLock.Unlock() if err := store.createTable(context.Background(), - path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)); err != nil { + path.Join(store.tablePathPrefix, bucket)); err != nil { glog.Errorf("createTable %s: %v", bucket, err) } @@ -319,7 +319,7 @@ func (store *YdbStore) OnBucketDeletion(bucket string) { defer store.dbsLock.Unlock() if err := store.deleteTable(context.Background(), - path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)); err != nil { + path.Join(store.tablePathPrefix, bucket)); err != nil { glog.Errorf("deleteTable %s: %v", bucket, err) } @@ -331,7 +331,7 @@ func (store *YdbStore) OnBucketDeletion(bucket string) { func (store *YdbStore) createTable(ctx context.Context, prefix string) error { return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - return s.CreateTable(ctx, prefix, createTableOptions()...) + return s.CreateTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE), createTableOptions()...) }) } @@ -340,7 +340,7 @@ func (store *YdbStore) deleteTable(ctx context.Context, prefix string) error { return nil } return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - return s.DropTable(ctx, prefix) + return s.DropTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE)) }) } @@ -366,15 +366,14 @@ func (store *YdbStore) getPrefix(ctx context.Context, dir string) (tablePathPref store.dbsLock.Lock() defer store.dbsLock.Unlock() + tablePathPrefix = path.Join(store.tablePathPrefix, bucket) if _, found := store.dbs[bucket]; !found { - if err := store.createTable(ctx, - path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)); err == nil { + if err := store.createTable(ctx, tablePathPrefix); err == nil { store.dbs[bucket] = true } else { - glog.Errorf("createTable %s: %v", bucket, err) + glog.Errorf("createTable %s: %v", tablePathPrefix, err) } } - tablePathPrefix = path.Join(store.tablePathPrefix, bucket) } return } diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index 069c35224..6b1e9b99a 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -3,6 +3,7 @@ package ydb import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/util" @@ -42,12 +43,14 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err return fmt.Errorf("kv get execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) } defer func() { _ = res.Close() }() - for res.NextRow() { - if err := res.ScanNamed(named.Required("meta", &value)); err != nil { - return fmt.Errorf("scanNamed %s : %v", util.NewFullPath(dirStr, name).Name(), err) + for res.NextResultSet(ctx) { + for res.NextRow() { + if err := res.ScanNamed(named.OptionalWithDefault("meta", &value)); err != nil { + return fmt.Errorf("scanNamed %s : %v", util.NewFullPath(dirStr, name).Name(), err) + } + valueFound = true + return nil } - valueFound = true - return nil } return res.Err() }) diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index 07e69e6a5..aab3d0f87 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -22,16 +22,16 @@ type FileMetas []FileMeta func (fm *FileMeta) queryParameters() *table.QueryParameters { return table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(fm.DirHash)), - table.ValueParam("$name", types.UTF8Value(fm.Name)), table.ValueParam("$directory", types.UTF8Value(fm.Directory)), + table.ValueParam("$name", types.UTF8Value(fm.Name)), table.ValueParam("$meta", types.StringValue(fm.Meta))) } func createTableOptions() []options.CreateTableOption { return []options.CreateTableOption{ options.WithColumn("dir_hash", types.Optional(types.TypeInt64)), - options.WithColumn("name", types.Optional(types.TypeUTF8)), options.WithColumn("directory", types.Optional(types.TypeUTF8)), + options.WithColumn("name", types.Optional(types.TypeUTF8)), options.WithColumn("meta", types.Optional(types.TypeString)), options.WithPrimaryKeyColumn("dir_hash", "name"), } From cb3c7a3cdb0c2f16603285bbc0a608635c161dc8 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 16:03:10 +0500 Subject: [PATCH 12/18] enable query cache policy instead of prepare --- weed/filer/ydb/ydb_queries.go | 9 +++------ weed/filer/ydb/ydb_store.go | 10 ++++------ weed/filer/ydb/ydb_store_kv.go | 33 ++++++++++++++------------------- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index bc2f37b10..63e98ebb7 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -23,16 +23,14 @@ const ( REPLACE INTO ` + asql.DEFAULT_TABLE + ` (dir_hash, name, directory, meta) VALUES - ($dir_hash, $name, $directory, $meta) - COMMIT;` + ($dir_hash, $name, $directory, $meta);` deleteQuery = ` DECLARE $dir_hash AS int64; DECLARE $name AS Utf8; DELETE FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash = $dir_hash AND name = $name; - COMMIT;` + WHERE dir_hash = $dir_hash AND name = $name;` findQuery = ` DECLARE $dir_hash AS int64; @@ -47,8 +45,7 @@ const ( DECLARE $directory AS Utf8; DELETE FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash = $dir_hash AND directory = $directory; - COMMIT;` + WHERE dir_hash = $dir_hash AND directory = $directory;` listDirectoryQuery = ` DECLARE $dir_hash AS int64; diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 678f58143..6e62d2719 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -12,6 +12,7 @@ import ( "github.com/ydb-platform/ydb-go-sdk/v3" "github.com/ydb-platform/ydb-go-sdk/v3/sugar" "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/options" "github.com/ydb-platform/ydb-go-sdk/v3/table/result" "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" "github.com/ydb-platform/ydb-go-sdk/v3/table/types" @@ -104,17 +105,14 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) { var res result.Result if tx, ok := ctx.Value("tx").(table.Transaction); ok { - res, err = tx.Execute(ctx, *query, params) + res, err = tx.Execute(ctx, *query, params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) if err != nil { return fmt.Errorf("execute transaction: %v", err) } } else { err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, *query) - if err != nil { - return fmt.Errorf("prepare: %v", err) - } - _, res, err = stmt.Execute(ctx, tc, params) + _, res, err = s.Execute(ctx, tc, *query, + params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) if err != nil { return fmt.Errorf("execute statement: %v", err) } diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index 6b1e9b99a..af147cf62 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -3,6 +3,7 @@ package ydb import ( "context" "fmt" + "github.com/ydb-platform/ydb-go-sdk/v3/table/options" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" @@ -16,11 +17,9 @@ func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err dirStr, dirHash, name := abstract_sql.GenDirAndName(key) fileMeta := FileMeta{dirHash, name, dirStr, value} return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), insertQuery)) - if err != nil { - return fmt.Errorf("kv put prepare %s: %v", util.NewFullPath(dirStr, name).Name(), err) - } - _, _, err = stmt.Execute(ctx, rwTX, fileMeta.queryParameters()) + _, _, err = s.Execute(ctx, rwTX, withPragma(store.getPrefix(ctx, dirStr), insertQuery), + fileMeta.queryParameters(), + options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) if err != nil { return fmt.Errorf("kv put execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) } @@ -32,13 +31,11 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err dirStr, dirHash, name := abstract_sql.GenDirAndName(key) valueFound := false err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), findQuery)) - if err != nil { - return fmt.Errorf("kv get prepare %s: %v", util.NewFullPath(dirStr, name), err) - } - _, res, err := stmt.Execute(ctx, roTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(dirHash)), - table.ValueParam("$name", types.UTF8Value(name)))) + _, res, err := s.Execute(ctx, roTX, withPragma(store.getPrefix(ctx, dirStr), findQuery), + table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(dirHash)), + table.ValueParam("$name", types.UTF8Value(name))), + options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) if err != nil { return fmt.Errorf("kv get execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) } @@ -65,13 +62,11 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { dirStr, dirHash, name := abstract_sql.GenDirAndName(key) return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - stmt, err := s.Prepare(ctx, withPragma(store.getPrefix(ctx, dirStr), insertQuery)) - if err != nil { - return fmt.Errorf("Prepare %s: %v", util.NewFullPath(dirStr, name).Name(), err) - } - _, _, err = stmt.Execute(ctx, rwTX, table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(dirHash)), - table.ValueParam("$name", types.UTF8Value(name)))) + _, _, err = s.Execute(ctx, rwTX, withPragma(store.getPrefix(ctx, dirStr), insertQuery), + table.NewQueryParameters( + table.ValueParam("$dir_hash", types.Int64Value(dirHash)), + table.ValueParam("$name", types.UTF8Value(name))), + options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) if err != nil { return fmt.Errorf("kv delete %s: %v", util.NewFullPath(dirStr, name).Name(), err) } From 04d5dff6bb974d49e166aafe52b415863b242ead Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 17:52:23 +0500 Subject: [PATCH 13/18] add support native ttl --- weed/filer/ydb/ydb_queries.go | 10 ++++++---- weed/filer/ydb/ydb_store.go | 2 +- weed/filer/ydb/ydb_store_kv.go | 2 +- weed/filer/ydb/ydb_types.go | 18 ++++++++++++++++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index 63e98ebb7..923ccc20f 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -8,22 +8,24 @@ const ( DECLARE $directory AS Utf8; DECLARE $name AS Utf8; DECLARE $meta AS String; + DECLARE $expire_at AS Optional; UPSERT INTO ` + asql.DEFAULT_TABLE + ` - (dir_hash, name, directory, meta) + (dir_hash, name, directory, meta, expire_at) VALUES - ($dir_hash, $name, $directory, $meta);` + ($dir_hash, $name, $directory, $meta, $expire_at);` updateQuery = ` DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; DECLARE $name AS Utf8; DECLARE $meta AS String; + DECLARE $expire_at AS Optional; REPLACE INTO ` + asql.DEFAULT_TABLE + ` - (dir_hash, name, directory, meta) + (dir_hash, name, directory, meta, expire_at) VALUES - ($dir_hash, $name, $directory, $meta);` + ($dir_hash, $name, $directory, $meta, $expire_at);` deleteQuery = ` DECLARE $dir_hash AS int64; diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 6e62d2719..e4aeabbb9 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -145,7 +145,7 @@ func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Ent } queryWithPragma := withPragma(store.getPrefix(ctx, dir), query) fileMeta := FileMeta{util.HashStringToLong(dir), name, dir, meta} - return store.doTxOrDB(ctx, &queryWithPragma, fileMeta.queryParameters(), rwTX, nil) + return store.doTxOrDB(ctx, &queryWithPragma, fileMeta.queryParameters(entry.TtlSec), rwTX, nil) } func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index af147cf62..d63b243e9 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -18,7 +18,7 @@ func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err fileMeta := FileMeta{dirHash, name, dirStr, value} return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { _, _, err = s.Execute(ctx, rwTX, withPragma(store.getPrefix(ctx, dirStr), insertQuery), - fileMeta.queryParameters(), + fileMeta.queryParameters(0), options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) if err != nil { return fmt.Errorf("kv put execute %s: %v", util.NewFullPath(dirStr, name).Name(), err) diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index aab3d0f87..6b89b7c57 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -19,21 +19,35 @@ type FileMeta struct { //ydb:gen scan,value type FileMetas []FileMeta -func (fm *FileMeta) queryParameters() *table.QueryParameters { +func (fm *FileMeta) queryParameters(ttlSec int32) *table.QueryParameters { + var expireAtValue types.Value + if ttlSec > 0 { + expireAtValue = types.Uint32Value(uint32(ttlSec)) + } else { + expireAtValue = types.NullValue(types.TypeUint32) + } return table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(fm.DirHash)), table.ValueParam("$directory", types.UTF8Value(fm.Directory)), table.ValueParam("$name", types.UTF8Value(fm.Name)), - table.ValueParam("$meta", types.StringValue(fm.Meta))) + table.ValueParam("$meta", types.StringValue(fm.Meta)), + table.ValueParam("$expire_at", expireAtValue)) } func createTableOptions() []options.CreateTableOption { + columnUnit := options.TimeToLiveUnitSeconds return []options.CreateTableOption{ options.WithColumn("dir_hash", types.Optional(types.TypeInt64)), options.WithColumn("directory", types.Optional(types.TypeUTF8)), options.WithColumn("name", types.Optional(types.TypeUTF8)), options.WithColumn("meta", types.Optional(types.TypeString)), + options.WithColumn("expire_at", types.Optional(types.TypeUint32)), options.WithPrimaryKeyColumn("dir_hash", "name"), + options.WithTimeToLiveSettings(options.TimeToLiveSettings{ + ColumnName: "expire_at", + ColumnUnit: &columnUnit, + Mode: options.TimeToLiveModeValueSinceUnixEpoch}, + ), } } func withPragma(prefix string, query string) string { From 7b544576af87865c42697492bfdf3af30ee27229 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 20:16:00 +0500 Subject: [PATCH 14/18] refactor --- weed/filer/ydb/ydb_queries.go | 21 +++++++- weed/filer/ydb/ydb_store.go | 97 +++++++++++++++++++++------------- weed/filer/ydb/ydb_store_kv.go | 6 +-- weed/filer/ydb/ydb_types.go | 6 ++- 4 files changed, 87 insertions(+), 43 deletions(-) diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index 923ccc20f..0f925584e 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -4,6 +4,7 @@ import asql "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" const ( insertQuery = ` + PRAGMA TablePathPrefix("%v"); DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; DECLARE $name AS Utf8; @@ -16,6 +17,7 @@ const ( ($dir_hash, $name, $directory, $meta, $expire_at);` updateQuery = ` + PRAGMA TablePathPrefix("%v"); DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; DECLARE $name AS Utf8; @@ -28,6 +30,7 @@ const ( ($dir_hash, $name, $directory, $meta, $expire_at);` deleteQuery = ` + PRAGMA TablePathPrefix("%v"); DECLARE $dir_hash AS int64; DECLARE $name AS Utf8; @@ -35,6 +38,7 @@ const ( WHERE dir_hash = $dir_hash AND name = $name;` findQuery = ` + PRAGMA TablePathPrefix("%v"); DECLARE $dir_hash AS int64; DECLARE $name AS Utf8; @@ -43,6 +47,7 @@ const ( WHERE dir_hash = $dir_hash AND name = $name;` deleteFolderChildrenQuery = ` + PRAGMA TablePathPrefix("%v"); DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; @@ -50,6 +55,7 @@ const ( WHERE dir_hash = $dir_hash AND directory = $directory;` listDirectoryQuery = ` + PRAGMA TablePathPrefix("%v"); DECLARE $dir_hash AS int64; DECLARE $directory AS Utf8; DECLARE $start_name AS Utf8; @@ -58,6 +64,19 @@ const ( SELECT name, meta FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash = $dir_hash AND directory = $directory and name %s $start_name and name LIKE $prefix + WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix + ORDER BY name ASC LIMIT $limit;` + + listInclusiveDirectoryQuery = ` + PRAGMA TablePathPrefix("%v"); + DECLARE $dir_hash AS int64; + DECLARE $directory AS Utf8; + DECLARE $start_name AS Utf8; + DECLARE $prefix AS Utf8; + DECLARE $limit AS Uint64; + + SELECT name, meta + FROM ` + asql.DEFAULT_TABLE + ` + WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix ORDER BY name ASC LIMIT $limit;` ) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index e4aeabbb9..3d40a0400 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -66,6 +66,9 @@ func (store *YdbStore) Initialize(configuration util.Configuration, prefix strin func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, connectionTimeOut int, poolSizeLimit int) (err error) { store.dirBuckets = dirBuckets store.SupportBucketTable = useBucketPrefix + if store.SupportBucketTable { + glog.V(0).Infof("enabled BucketPrefix") + } store.dbs = make(map[string]bool) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -133,7 +136,7 @@ func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *tabl return err } -func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Entry, query string) (err error) { +func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Entry, isUpdate bool) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() if err != nil { @@ -143,29 +146,36 @@ func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Ent if len(entry.Chunks) > filer.CountEntryChunksForGzip { meta = util.MaybeGzipData(meta) } - queryWithPragma := withPragma(store.getPrefix(ctx, dir), query) - fileMeta := FileMeta{util.HashStringToLong(dir), name, dir, meta} - return store.doTxOrDB(ctx, &queryWithPragma, fileMeta.queryParameters(entry.TtlSec), rwTX, nil) + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + fileMeta := FileMeta{util.HashStringToLong(dir), name, *shortDir, meta} + var query *string + if isUpdate { + query = withPragma(tablePathPrefix, updateQuery) + } else { + query = withPragma(tablePathPrefix, insertQuery) + } + return store.doTxOrDB(ctx, query, fileMeta.queryParameters(entry.TtlSec), rwTX, nil) } func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { - return store.insertOrUpdateEntry(ctx, entry, insertQuery) + return store.insertOrUpdateEntry(ctx, entry, false) } func (store *YdbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { - return store.insertOrUpdateEntry(ctx, entry, updateQuery) + return store.insertOrUpdateEntry(ctx, entry, true) } func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { dir, name := fullpath.DirAndName() var data []byte entryFound := false - queryWithPragma := withPragma(store.getPrefix(ctx, dir), findQuery) + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + query := withPragma(tablePathPrefix, findQuery) queryParams := table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), table.ValueParam("$name", types.UTF8Value(name))) - err = store.doTxOrDB(ctx, &queryWithPragma, queryParams, roTX, func(res result.Result) error { + err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error { for res.NextResultSet(ctx) { for res.NextRow() { if err = res.ScanNamed(named.OptionalWithDefault("meta", &data)); err != nil { @@ -196,22 +206,24 @@ func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (e func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { dir, name := fullpath.DirAndName() - queryWithPragma := withPragma(store.getPrefix(ctx, dir), deleteQuery) + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + query := withPragma(tablePathPrefix, deleteQuery) queryParams := table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), table.ValueParam("$name", types.UTF8Value(name))) - return store.doTxOrDB(ctx, &queryWithPragma, queryParams, rwTX, nil) + return store.doTxOrDB(ctx, query, queryParams, rwTX, nil) } func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { dir, _ := fullpath.DirAndName() - queryWithPragma := withPragma(store.getPrefix(ctx, dir), deleteFolderChildrenQuery) + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + query := withPragma(tablePathPrefix, deleteFolderChildrenQuery) queryParams := table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$directory", types.UTF8Value(dir))) + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), + table.ValueParam("$directory", types.UTF8Value(*shortDir))) - return store.doTxOrDB(ctx, &queryWithPragma, queryParams, rwTX, nil) + return store.doTxOrDB(ctx, query, queryParams, rwTX, nil) } func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { @@ -220,19 +232,21 @@ func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.Fu func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { dir := string(dirPath) - startFileCompOp := ">" + tablePathPrefix, shortDir := store.getPrefix(ctx, &dir) + var query *string if includeStartFile { - startFileCompOp = ">=" + query = withPragma(tablePathPrefix, listInclusiveDirectoryQuery) + } else { + query = withPragma(tablePathPrefix, listDirectoryQuery) } - queryWithPragma := withPragma(store.getPrefix(ctx, dir), fmt.Sprintf(listDirectoryQuery, startFileCompOp)) queryParams := table.NewQueryParameters( - table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(dir))), - table.ValueParam("$directory", types.UTF8Value(dir)), + table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), + table.ValueParam("$directory", types.UTF8Value(*shortDir)), table.ValueParam("$start_name", types.UTF8Value(startFileName)), table.ValueParam("$prefix", types.UTF8Value(prefix+"%")), table.ValueParam("$limit", types.Uint64Value(uint64(limit))), ) - err = store.doTxOrDB(ctx, &queryWithPragma, queryParams, roTX, func(res result.Result) error { + err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error { var name string var data []byte for res.NextResultSet(ctx) { @@ -337,41 +351,50 @@ func (store *YdbStore) deleteTable(ctx context.Context, prefix string) error { if !store.SupportBucketTable { return nil } - return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { + if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { return s.DropTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE)) - }) + }); err != nil { + return err + } + glog.V(4).Infof("deleted table %s", prefix) + + return nil } -func (store *YdbStore) getPrefix(ctx context.Context, dir string) (tablePathPrefix string) { - tablePathPrefix = store.tablePathPrefix +func (store *YdbStore) getPrefix(ctx context.Context, dir *string) (tablePathPrefix *string, shortDir *string) { + tablePathPrefix = &store.tablePathPrefix + shortDir = dir if !store.SupportBucketTable { return } prefixBuckets := store.dirBuckets + "/" - if strings.HasPrefix(dir, prefixBuckets) { + if strings.HasPrefix(*dir, prefixBuckets) { // detect bucket - bucketAndDir := dir[len(prefixBuckets):] - t := strings.Index(bucketAndDir, "/") - if t < 0 { + bucketAndDir := (*dir)[len(prefixBuckets):] + var bucket string + if t := strings.Index(bucketAndDir, "/"); t > 0 { + bucket = bucketAndDir[:t] + } else if t < 0 { + bucket = bucketAndDir + } + if bucket == "" { return } - bucket := bucketAndDir[:t] - if bucket != "" { - return - } store.dbsLock.Lock() defer store.dbsLock.Unlock() - tablePathPrefix = path.Join(store.tablePathPrefix, bucket) + tablePathPrefixWithBucket := path.Join(store.tablePathPrefix, bucket) if _, found := store.dbs[bucket]; !found { - if err := store.createTable(ctx, tablePathPrefix); err == nil { + if err := store.createTable(ctx, tablePathPrefixWithBucket); err == nil { store.dbs[bucket] = true + glog.V(4).Infof("created table %s", tablePathPrefixWithBucket) } else { - glog.Errorf("createTable %s: %v", tablePathPrefix, err) + glog.Errorf("createTable %s: %v", tablePathPrefixWithBucket, err) } } + tablePathPrefix = &tablePathPrefixWithBucket } return } diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index d63b243e9..c90821ddd 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -17,7 +17,7 @@ func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err dirStr, dirHash, name := abstract_sql.GenDirAndName(key) fileMeta := FileMeta{dirHash, name, dirStr, value} return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - _, _, err = s.Execute(ctx, rwTX, withPragma(store.getPrefix(ctx, dirStr), insertQuery), + _, _, err = s.Execute(ctx, rwTX, *withPragma(&store.tablePathPrefix, insertQuery), fileMeta.queryParameters(0), options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) if err != nil { @@ -31,7 +31,7 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err dirStr, dirHash, name := abstract_sql.GenDirAndName(key) valueFound := false err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error { - _, res, err := s.Execute(ctx, roTX, withPragma(store.getPrefix(ctx, dirStr), findQuery), + _, res, err := s.Execute(ctx, roTX, *withPragma(&store.tablePathPrefix, findQuery), table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(dirHash)), table.ValueParam("$name", types.UTF8Value(name))), @@ -62,7 +62,7 @@ func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) { dirStr, dirHash, name := abstract_sql.GenDirAndName(key) return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - _, _, err = s.Execute(ctx, rwTX, withPragma(store.getPrefix(ctx, dirStr), insertQuery), + _, _, err = s.Execute(ctx, rwTX, *withPragma(&store.tablePathPrefix, insertQuery), table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(dirHash)), table.ValueParam("$name", types.UTF8Value(name))), diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index 6b89b7c57..34044b6a7 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -1,6 +1,7 @@ package ydb import ( + "fmt" "github.com/ydb-platform/ydb-go-sdk/v3/table" "github.com/ydb-platform/ydb-go-sdk/v3/table/options" "github.com/ydb-platform/ydb-go-sdk/v3/table/types" @@ -50,6 +51,7 @@ func createTableOptions() []options.CreateTableOption { ), } } -func withPragma(prefix string, query string) string { - return `PRAGMA TablePathPrefix("` + prefix + `");` + query +func withPragma(prefix *string, query string) *string { + queryWithPragma := fmt.Sprintf(query, *prefix) + return &queryWithPragma } From f127b326bfca03df506fd8481c74ab874422098d Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 22:54:31 +0500 Subject: [PATCH 15/18] add options to scaffold --- weed/command/scaffold/filer.toml | 12 +++++++----- weed/filer/ydb/readme.md | 1 + weed/filer/ydb/ydb_store.go | 12 ++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 3a920d80b..595fb2e62 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -295,12 +295,14 @@ password="" # skip tls cert validation insecure_skip_verify = true -[ydb] +[ydb] # https://ydb.tech/ enabled = false -useBucketPrefix=true -dsn="grpc://localhost:2136?database=/local" -prefix="en" -poolSizeLimit=50 +dsn = "grpc://localhost:2136?database=/local" +prefix = "seaweedfs" +useBucketPrefix = true # Fast Bucket Deletion +poolSizeLimit = 50 +dialTimeOut = 10 + # Authenticate produced with one of next environment variables: # YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS= — used service account key file by path # YDB_ANONYMOUS_CREDENTIALS="1" — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation diff --git a/weed/filer/ydb/readme.md b/weed/filer/ydb/readme.md index fd6c5c0a2..b617461fd 100644 --- a/weed/filer/ydb/readme.md +++ b/weed/filer/ydb/readme.md @@ -13,6 +13,7 @@ dsn=grpcs://ydb-ru.yandex.net:2135/?database=/ru/home/username/db prefix="seaweedfs" useBucketPrefix=true poolSizeLimit=50 +dialTimeOut = 10 ``` Authenticate produced with one of next environment variables: diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 3d40a0400..026b34b4b 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -24,7 +24,7 @@ import ( ) const ( - defaultConnectionTimeOut = 10 + defaultDialTimeOut = 10 ) var ( @@ -58,12 +58,12 @@ func (store *YdbStore) Initialize(configuration util.Configuration, prefix strin configuration.GetString(prefix+"dsn"), configuration.GetString(prefix+"prefix"), configuration.GetBool(prefix+"useBucketPrefix"), - configuration.GetInt(prefix+"connectionTimeOut"), + configuration.GetInt(prefix+"dialTimeOut"), configuration.GetInt(prefix+"poolSizeLimit"), ) } -func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, connectionTimeOut int, poolSizeLimit int) (err error) { +func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, dialTimeOut int, poolSizeLimit int) (err error) { store.dirBuckets = dirBuckets store.SupportBucketTable = useBucketPrefix if store.SupportBucketTable { @@ -72,11 +72,11 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix store.dbs = make(map[string]bool) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if connectionTimeOut == 0 { - connectionTimeOut = defaultConnectionTimeOut + if dialTimeOut == 0 { + dialTimeOut = defaultDialTimeOut } opts := []ydb.Option{ - ydb.WithDialTimeout(time.Duration(connectionTimeOut) * time.Second), + ydb.WithDialTimeout(time.Duration(dialTimeOut) * time.Second), environ.WithEnvironCredentials(ctx), } if poolSizeLimit > 0 { From 7d18b2756d82c240548eb5f581d6e36f5f2232fc Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 23:06:41 +0500 Subject: [PATCH 16/18] SeaweedFS Filer readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 13c23d577..b04c5188c 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Also, SeaweedFS implements erasure coding with ideas from On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, -e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, etc. +e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, YDB, etc. For any distributed key value stores, the large values can be offloaded to SeaweedFS. With the fast access speed and linearly scalable capacity, @@ -412,7 +412,7 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa * SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files. * SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached. -* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized. +* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd, YDB etc, and is easy to customized. * SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc. | System | File Metadata | File Content Read| POSIX | REST API | Optimized for large number of small files | @@ -454,7 +454,7 @@ Ceph uses CRUSH hashing to automatically manage the data placement, which is eff SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read. -SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage. +SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, YDB, to manage file directories. These stores are proven, scalable, and easier to manage. | SeaweedFS | comparable to Ceph | advantage | | ------------- | ------------- | ---------------- | From f58adaab2594298838ae4184de594e89ca7471b6 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 3 May 2022 23:20:09 +0500 Subject: [PATCH 17/18] build tag ydb since the growth of the binary is 5 mb --- .github/workflows/go.yml | 4 ++-- Makefile | 2 +- weed/filer/ydb/ydb_store.go | 3 +++ weed/filer/ydb/ydb_store_kv.go | 3 +++ weed/filer/ydb/ydb_types.go | 3 +++ 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 96632fc86..f40e759d3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -34,7 +34,7 @@ jobs: cd weed; go get -v -t -d ./... - name: Build - run: cd weed; go build -tags "elastic gocdk sqlite hdfs" -v . + run: cd weed; go build -tags "elastic gocdk sqlite hdfs ydb" -v . - name: Test - run: cd weed; go test -tags "elastic gocdk sqlite hdfs" -v ./... + run: cd weed; go test -tags "elastic gocdk sqlite hdfs ydb" -v ./... diff --git a/Makefile b/Makefile index 36da78434..1287d9f8d 100644 --- a/Makefile +++ b/Makefile @@ -8,4 +8,4 @@ install: cd weed; go install full_install: - cd weed; go install -tags "elastic gocdk sqlite hdfs" + cd weed; go install -tags "elastic gocdk sqlite hdfs ydb" diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index 026b34b4b..5b0e4e764 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -1,3 +1,6 @@ +//go:build ydb +// +build ydb + package ydb import ( diff --git a/weed/filer/ydb/ydb_store_kv.go b/weed/filer/ydb/ydb_store_kv.go index c90821ddd..d64597764 100644 --- a/weed/filer/ydb/ydb_store_kv.go +++ b/weed/filer/ydb/ydb_store_kv.go @@ -1,3 +1,6 @@ +//go:build ydb +// +build ydb + package ydb import ( diff --git a/weed/filer/ydb/ydb_types.go b/weed/filer/ydb/ydb_types.go index 34044b6a7..d256eaf50 100644 --- a/weed/filer/ydb/ydb_types.go +++ b/weed/filer/ydb/ydb_types.go @@ -1,3 +1,6 @@ +//go:build ydb +// +build ydb + package ydb import ( From e6d2cb59111e73d97ecaf2c02d4a6d7d04f6fc3d Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 5 May 2022 11:12:14 +0500 Subject: [PATCH 18/18] ydb add empty doc file --- weed/filer/ydb/doc.go | 9 +++++++++ weed/filer/ydb/ydb_queries.go | 3 +++ 2 files changed, 12 insertions(+) create mode 100644 weed/filer/ydb/doc.go diff --git a/weed/filer/ydb/doc.go b/weed/filer/ydb/doc.go new file mode 100644 index 000000000..6ade3a8d8 --- /dev/null +++ b/weed/filer/ydb/doc.go @@ -0,0 +1,9 @@ +/* + +Package ydb is for YDB filer store. + +The referenced "github.com/ydb-platform/ydb-go-sdk/v3" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package ydb diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index 0f925584e..c8876e004 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -1,3 +1,6 @@ +//go:build ydb +// +build ydb + package ydb import asql "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"