From d12a423aa0c6d8244d4a46ed2ec8804ad64f2081 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 12 May 2022 12:23:02 +0500 Subject: [PATCH] refactor add some tests https://github.com/chrislusf/seaweedfs/pull/2996 --- docker/Makefile | 10 ++++++---- ...cal-ydb-compose.yml => test-ydb-filer.yml} | 6 +++--- weed/filer/ydb/ydb_store.go | 16 +++------------- weed/filer/ydb/ydb_store_test.go | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 20 deletions(-) rename docker/compose/{local-ydb-compose.yml => test-ydb-filer.yml} (91%) create mode 100644 weed/filer/ydb/ydb_store_test.go diff --git a/docker/Makefile b/docker/Makefile index 1ace31933..c8603309d 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -7,7 +7,7 @@ gen: dev binary: export SWCOMMIT=$(shell git rev-parse --short HEAD) export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)" - cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/ + cd ../weed; CGO_ENABLED=0 GOOS=linux go build --tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/ build: binary docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local . @@ -49,9 +49,6 @@ dev_replicate: build dev_auditlog: build docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up -dev_ydb: build - docker-compose -f compose/local-ydb-compose.yml -p seaweedfs up - cluster: build docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up @@ -73,6 +70,11 @@ filer_etcd: build test_etcd: build docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up +test_ydb: tags = ydb +test_ydb: build + export + docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up + clean: rm ./weed diff --git a/docker/compose/local-ydb-compose.yml b/docker/compose/test-ydb-filer.yml similarity index 91% rename from docker/compose/local-ydb-compose.yml rename to docker/compose/test-ydb-filer.yml index a17b77b8a..c0c31fe5b 100644 --- a/docker/compose/local-ydb-compose.yml +++ b/docker/compose/test-ydb-filer.yml @@ -17,8 +17,6 @@ services: ports: - 9333:9333 - 19333:19333 - - 8084:8080 - - 18084:18080 - 8888:8888 - 8000:8000 - 18888:18888 @@ -32,4 +30,6 @@ services: WEED_YDB_PREFIX: "seaweedfs" YDB_ANONYMOUS_CREDENTIALS: 1 WEED_MASTER_VOLUME_GROWTH_COPY_1: 1 - WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 \ No newline at end of file + WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1 + depends_on: + - ydb \ No newline at end of file diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index b06b35ac2..1e3a55a09 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -15,7 +15,6 @@ import ( "github.com/ydb-platform/ydb-go-sdk/v3" "github.com/ydb-platform/ydb-go-sdk/v3/sugar" "github.com/ydb-platform/ydb-go-sdk/v3/table" - "github.com/ydb-platform/ydb-go-sdk/v3/table/options" "github.com/ydb-platform/ydb-go-sdk/v3/table/result" "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" "github.com/ydb-platform/ydb-go-sdk/v3/table/types" @@ -28,7 +27,6 @@ import ( const ( defaultDialTimeOut = 10 - maxRowsInQuery = 1000 // Limit number of rows in query results https://cloud.yandex.com/en-ru/docs/ydb/concepts/limits-ydb ) var ( @@ -112,14 +110,13 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) { var res result.Result if tx, ok := ctx.Value("tx").(table.Transaction); ok { - res, err = tx.Execute(ctx, *query, params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) + res, err = tx.Execute(ctx, *query, params) if err != nil { return fmt.Errorf("execute transaction: %v", err) } } else { err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { - _, res, err = s.Execute(ctx, tc, *query, - params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache())) + _, res, err = s.Execute(ctx, tc, *query, params) if err != nil { return fmt.Errorf("execute statement: %v", err) } @@ -240,10 +237,6 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath } truncated := true eachEntryFuncIsNotBreake := true - shortLimit := limit - if limit > maxRowsInQuery { - shortLimit = maxRowsInQuery * 2 - } entryCount := int64(0) for truncated && eachEntryFuncIsNotBreake { if lastFileName != "" { @@ -253,15 +246,12 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath } } restLimit := limit - entryCount - if maxRowsInQuery > restLimit { - shortLimit = restLimit - } queryParams := table.NewQueryParameters( table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), table.ValueParam("$directory", types.UTF8Value(*shortDir)), table.ValueParam("$start_name", types.UTF8Value(startFileName)), table.ValueParam("$prefix", types.UTF8Value(prefix+"%")), - table.ValueParam("$limit", types.Uint64Value(uint64(shortLimit))), + table.ValueParam("$limit", types.Uint64Value(uint64(restLimit))), ) err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error { var name string diff --git a/weed/filer/ydb/ydb_store_test.go b/weed/filer/ydb/ydb_store_test.go new file mode 100644 index 000000000..cb3c77018 --- /dev/null +++ b/weed/filer/ydb/ydb_store_test.go @@ -0,0 +1,19 @@ +//go:build ydb +// +build ydb + +package ydb + +import ( + "github.com/chrislusf/seaweedfs/weed/filer/store_test" + "testing" +) + +func TestStore(t *testing.T) { + // run "make test_ydb" under docker folder. + // to set up local env + if false { + store := &YdbStore{} + store.initialize("/buckets", "grpc://localhost:2136/?database=local", "seaweedfs", true, 10, 50) + store_test.TestFilerStore(t, store) + } +}