Merge pull request #3043 from kmlebedev/ydb_refactor

ydb test&refactor
This commit is contained in:
Chris Lu 2022-05-12 00:50:07 -07:00 committed by GitHub
commit 8f103ae613
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 31 additions and 20 deletions

View file

@ -7,7 +7,7 @@ gen: dev
binary:
export SWCOMMIT=$(shell git rev-parse --short HEAD)
export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
cd ../weed; CGO_ENABLED=0 GOOS=linux go build --tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
build: binary
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
@ -49,9 +49,6 @@ dev_replicate: build
dev_auditlog: build
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
dev_ydb: build
docker-compose -f compose/local-ydb-compose.yml -p seaweedfs up
cluster: build
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
@ -73,6 +70,11 @@ filer_etcd: build
test_etcd: build
docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
test_ydb: tags = ydb
test_ydb: build
export
docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
clean:
rm ./weed

View file

@ -17,8 +17,6 @@ services:
ports:
- 9333:9333
- 19333:19333
- 8084:8080
- 18084:18080
- 8888:8888
- 8000:8000
- 18888:18888
@ -32,4 +30,6 @@ services:
WEED_YDB_PREFIX: "seaweedfs"
YDB_ANONYMOUS_CREDENTIALS: 1
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
depends_on:
- ydb

View file

@ -15,7 +15,6 @@ import (
"github.com/ydb-platform/ydb-go-sdk/v3"
"github.com/ydb-platform/ydb-go-sdk/v3/sugar"
"github.com/ydb-platform/ydb-go-sdk/v3/table"
"github.com/ydb-platform/ydb-go-sdk/v3/table/options"
"github.com/ydb-platform/ydb-go-sdk/v3/table/result"
"github.com/ydb-platform/ydb-go-sdk/v3/table/result/named"
"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
@ -28,7 +27,6 @@ import (
const (
defaultDialTimeOut = 10
maxRowsInQuery = 1000 // Limit number of rows in query results https://cloud.yandex.com/en-ru/docs/ydb/concepts/limits-ydb
)
var (
@ -112,14 +110,13 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix
func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) {
var res result.Result
if tx, ok := ctx.Value("tx").(table.Transaction); ok {
res, err = tx.Execute(ctx, *query, params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache()))
res, err = tx.Execute(ctx, *query, params)
if err != nil {
return fmt.Errorf("execute transaction: %v", err)
}
} else {
err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) {
_, res, err = s.Execute(ctx, tc, *query,
params, options.WithQueryCachePolicy(options.WithQueryCachePolicyKeepInCache()))
_, res, err = s.Execute(ctx, tc, *query, params)
if err != nil {
return fmt.Errorf("execute statement: %v", err)
}
@ -240,10 +237,6 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath
}
truncated := true
eachEntryFuncIsNotBreake := true
shortLimit := limit
if limit > maxRowsInQuery {
shortLimit = maxRowsInQuery * 2
}
entryCount := int64(0)
for truncated && eachEntryFuncIsNotBreake {
if lastFileName != "" {
@ -253,15 +246,12 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath
}
}
restLimit := limit - entryCount
if maxRowsInQuery > restLimit {
shortLimit = restLimit
}
queryParams := table.NewQueryParameters(
table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
table.ValueParam("$directory", types.UTF8Value(*shortDir)),
table.ValueParam("$start_name", types.UTF8Value(startFileName)),
table.ValueParam("$prefix", types.UTF8Value(prefix+"%")),
table.ValueParam("$limit", types.Uint64Value(uint64(shortLimit))),
table.ValueParam("$limit", types.Uint64Value(uint64(restLimit))),
)
err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error {
var name string

View file

@ -0,0 +1,19 @@
//go:build ydb
// +build ydb
package ydb
import (
"github.com/chrislusf/seaweedfs/weed/filer/store_test"
"testing"
)
func TestStore(t *testing.T) {
// run "make test_ydb" under docker folder.
// to set up local env
if false {
store := &YdbStore{}
store.initialize("/buckets", "grpc://localhost:2136/?database=local", "seaweedfs", true, 10, 50)
store_test.TestFilerStore(t, store)
}
}