seaweedfs/weed/filer/ydb/ydb_store.go

416 lines
12 KiB
Go
Raw Normal View History

//go:build ydb
// +build ydb
package ydb
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
2022-05-02 17:23:07 +00:00
environ "github.com/ydb-platform/ydb-go-sdk-auth-environ"
2022-05-02 07:42:20 +00:00
"github.com/ydb-platform/ydb-go-sdk/v3"
2022-05-01 21:07:47 +00:00
"github.com/ydb-platform/ydb-go-sdk/v3/sugar"
"github.com/ydb-platform/ydb-go-sdk/v3/table"
2022-05-02 10:33:29 +00:00
"github.com/ydb-platform/ydb-go-sdk/v3/table/result"
2022-05-01 21:07:47 +00:00
"github.com/ydb-platform/ydb-go-sdk/v3/table/result/named"
"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
2022-05-02 07:42:20 +00:00
"os"
2022-05-01 15:33:03 +00:00
"path"
"strings"
2022-05-02 07:42:20 +00:00
"sync"
"time"
)
const (
2022-05-03 17:54:31 +00:00
defaultDialTimeOut = 10
)
var (
roTX = table.TxControl(
table.BeginTx(table.WithOnlineReadOnly()),
table.CommitTx(),
)
2022-05-01 21:07:47 +00:00
rwTX = table.DefaultTxControl()
)
type YdbStore struct {
2022-05-01 21:07:47 +00:00
DB ydb.Connection
dirBuckets string
tablePathPrefix string
2022-05-02 07:42:20 +00:00
SupportBucketTable bool
dbs map[string]bool
dbsLock sync.Mutex
}
func init() {
filer.Stores = append(filer.Stores, &YdbStore{})
}
func (store *YdbStore) GetName() string {
return "ydb"
}
func (store *YdbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize(
configuration.GetString("filer.options.buckets_folder"),
2022-05-01 21:07:47 +00:00
configuration.GetString(prefix+"dsn"),
2022-05-03 10:18:28 +00:00
configuration.GetString(prefix+"prefix"),
configuration.GetBool(prefix+"useBucketPrefix"),
2022-05-03 17:54:31 +00:00
configuration.GetInt(prefix+"dialTimeOut"),
2022-05-01 21:07:47 +00:00
configuration.GetInt(prefix+"poolSizeLimit"),
)
}
2022-05-03 17:54:31 +00:00
func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, dialTimeOut int, poolSizeLimit int) (err error) {
store.dirBuckets = dirBuckets
store.SupportBucketTable = useBucketPrefix
2022-05-03 15:16:00 +00:00
if store.SupportBucketTable {
glog.V(0).Infof("enabled BucketPrefix")
}
2022-05-02 07:42:20 +00:00
store.dbs = make(map[string]bool)
2022-05-01 21:07:47 +00:00
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
2022-05-03 17:54:31 +00:00
if dialTimeOut == 0 {
dialTimeOut = defaultDialTimeOut
}
2022-05-01 21:07:47 +00:00
opts := []ydb.Option{
2022-05-03 17:54:31 +00:00
ydb.WithDialTimeout(time.Duration(dialTimeOut) * time.Second),
2022-05-02 17:23:07 +00:00
environ.WithEnvironCredentials(ctx),
2022-05-01 21:07:47 +00:00
}
if poolSizeLimit > 0 {
opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit))
}
2022-05-02 07:42:20 +00:00
if dsn == "" {
dsn = os.Getenv("YDB_CONNECTION_STRING")
}
2022-05-01 21:07:47 +00:00
store.DB, err = ydb.Open(ctx, dsn, opts...)
if err != nil {
2022-05-02 17:23:07 +00:00
if store.DB != nil {
_ = store.DB.Close(ctx)
store.DB = nil
}
return fmt.Errorf("can not connect to %s error: %v", dsn, err)
}
2022-05-02 07:42:20 +00:00
2022-05-01 21:07:47 +00:00
store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix)
if err = sugar.MakeRecursive(ctx, store.DB, store.tablePathPrefix); err != nil {
return fmt.Errorf("MakeRecursive %s : %v", store.tablePathPrefix, err)
}
2022-05-02 07:42:20 +00:00
2022-05-03 10:18:28 +00:00
if err = store.createTable(ctx, store.tablePathPrefix); err != nil {
glog.Errorf("createTable %s: %v", store.tablePathPrefix, err)
2022-05-02 07:42:20 +00:00
}
2022-05-02 17:23:07 +00:00
return err
}
2022-05-02 10:33:29 +00:00
func (store *YdbStore) doTxOrDB(ctx context.Context, query *string, params *table.QueryParameters, tc *table.TransactionControl, processResultFunc func(res result.Result) error) (err error) {
var res result.Result
if tx, ok := ctx.Value("tx").(table.Transaction); ok {
res, err = tx.Execute(ctx, *query, params)
2022-05-02 10:33:29 +00:00
if err != nil {
return fmt.Errorf("execute transaction: %v", err)
}
} else {
err = store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) {
_, res, err = s.Execute(ctx, tc, *query, params)
2022-05-02 10:33:29 +00:00
if err != nil {
return fmt.Errorf("execute statement: %v", err)
}
return nil
})
}
2022-05-03 10:18:28 +00:00
if err != nil {
return err
}
if res != nil {
defer func() { _ = res.Close() }()
if processResultFunc != nil {
if err = processResultFunc(res); err != nil {
return fmt.Errorf("process result: %v", err)
}
2022-05-02 10:33:29 +00:00
}
}
return err
}
func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
dir, name := entry.FullPath.DirAndName()
meta, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
}
if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
meta = util.MaybeGzipData(meta)
}
2022-05-03 15:16:00 +00:00
tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
fileMeta := FileMeta{util.HashStringToLong(dir), name, *shortDir, meta}
return store.doTxOrDB(ctx, withPragma(tablePathPrefix, upsertQuery), fileMeta.queryParameters(entry.TtlSec), rwTX, nil)
}
func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
return store.insertOrUpdateEntry(ctx, entry)
}
func (store *YdbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
return store.insertOrUpdateEntry(ctx, entry)
}
func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
dir, name := fullpath.DirAndName()
2022-05-01 21:07:47 +00:00
var data []byte
entryFound := false
2022-05-03 15:16:00 +00:00
tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
query := withPragma(tablePathPrefix, findQuery)
2022-05-02 10:33:29 +00:00
queryParams := table.NewQueryParameters(
2022-05-03 15:16:00 +00:00
table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
2022-05-02 10:33:29 +00:00
table.ValueParam("$name", types.UTF8Value(name)))
2022-05-03 15:16:00 +00:00
err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error {
if !res.NextResultSet(ctx) || !res.HasNextRow() {
return nil
}
for res.NextRow() {
if err = res.ScanNamed(named.OptionalWithDefault("meta", &data)); err != nil {
return fmt.Errorf("scanNamed %s : %v", fullpath, err)
}
entryFound = true
return nil
2022-05-01 21:07:47 +00:00
}
return res.Err()
})
if err != nil {
return nil, err
}
2022-05-01 21:07:47 +00:00
if !entryFound {
return nil, filer_pb.ErrNotFound
}
2022-05-03 10:18:28 +00:00
entry = &filer.Entry{
FullPath: fullpath,
}
2022-05-01 21:07:47 +00:00
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
2022-05-03 10:18:28 +00:00
return nil, fmt.Errorf("decode %s : %v", fullpath, err)
}
2022-05-01 21:07:47 +00:00
return entry, nil
}
func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
dir, name := fullpath.DirAndName()
2022-05-03 15:16:00 +00:00
tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
query := withPragma(tablePathPrefix, deleteQuery)
2022-05-02 10:33:29 +00:00
queryParams := table.NewQueryParameters(
2022-05-03 15:16:00 +00:00
table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
2022-05-02 10:33:29 +00:00
table.ValueParam("$name", types.UTF8Value(name)))
2022-05-03 15:16:00 +00:00
return store.doTxOrDB(ctx, query, queryParams, rwTX, nil)
}
func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
2022-05-01 15:33:03 +00:00
dir, _ := fullpath.DirAndName()
2022-05-03 15:16:00 +00:00
tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
query := withPragma(tablePathPrefix, deleteFolderChildrenQuery)
2022-05-02 10:33:29 +00:00
queryParams := table.NewQueryParameters(
2022-05-03 15:16:00 +00:00
table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
table.ValueParam("$directory", types.UTF8Value(*shortDir)))
2022-05-02 10:33:29 +00:00
2022-05-03 15:16:00 +00:00
return store.doTxOrDB(ctx, query, queryParams, rwTX, nil)
}
func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
}
func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
2022-05-01 15:33:03 +00:00
dir := string(dirPath)
2022-05-03 15:16:00 +00:00
tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
var query *string
2022-05-01 15:33:03 +00:00
if includeStartFile {
2022-05-03 15:16:00 +00:00
query = withPragma(tablePathPrefix, listInclusiveDirectoryQuery)
} else {
query = withPragma(tablePathPrefix, listDirectoryQuery)
2022-05-01 15:33:03 +00:00
}
truncated := true
eachEntryFuncIsNotBreake := true
entryCount := int64(0)
for truncated && eachEntryFuncIsNotBreake {
if lastFileName != "" {
startFileName = lastFileName
if includeStartFile {
query = withPragma(tablePathPrefix, listDirectoryQuery)
}
}
restLimit := limit - entryCount
queryParams := table.NewQueryParameters(
table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
table.ValueParam("$directory", types.UTF8Value(*shortDir)),
table.ValueParam("$start_name", types.UTF8Value(startFileName)),
table.ValueParam("$prefix", types.UTF8Value(prefix+"%")),
table.ValueParam("$limit", types.Uint64Value(uint64(restLimit))),
)
err = store.doTxOrDB(ctx, query, queryParams, roTX, func(res result.Result) error {
var name string
var data []byte
if !res.NextResultSet(ctx) || !res.HasNextRow() {
truncated = false
return nil
}
truncated = res.CurrentResultSet().Truncated()
2022-05-01 21:07:47 +00:00
for res.NextRow() {
if err := res.ScanNamed(
2022-05-03 10:18:28 +00:00
named.OptionalWithDefault("name", &name),
named.OptionalWithDefault("meta", &data)); err != nil {
return fmt.Errorf("list scanNamed %s : %v", dir, err)
2022-05-01 21:07:47 +00:00
}
lastFileName = name
entry := &filer.Entry{
FullPath: util.NewFullPath(dir, name),
}
if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
return fmt.Errorf("scan decode %s : %v", entry.FullPath, err)
}
if !eachEntryFunc(entry) {
eachEntryFuncIsNotBreake = false
2022-05-01 21:07:47 +00:00
break
}
entryCount += 1
2022-05-01 15:33:03 +00:00
}
return res.Err()
})
}
2022-05-01 15:33:03 +00:00
if err != nil {
return lastFileName, err
}
return lastFileName, nil
}
func (store *YdbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
2022-05-01 21:07:47 +00:00
session, err := store.DB.Table().CreateSession(ctx)
2022-05-01 15:33:03 +00:00
if err != nil {
return ctx, err
}
tx, err := session.BeginTransaction(ctx, table.TxSettings(table.WithSerializableReadWrite()))
if err != nil {
return ctx, err
}
return context.WithValue(ctx, "tx", tx), nil
}
func (store *YdbStore) CommitTransaction(ctx context.Context) error {
2022-05-01 21:07:47 +00:00
if tx, ok := ctx.Value("tx").(table.Transaction); ok {
_, err := tx.CommitTx(ctx)
return err
2022-05-01 15:33:03 +00:00
}
return nil
}
func (store *YdbStore) RollbackTransaction(ctx context.Context) error {
2022-05-01 21:07:47 +00:00
if tx, ok := ctx.Value("tx").(table.Transaction); ok {
2022-05-01 15:33:03 +00:00
return tx.Rollback(ctx)
}
return nil
}
func (store *YdbStore) Shutdown() {
2022-05-01 21:07:47 +00:00
_ = store.DB.Close(context.Background())
}
2022-07-22 01:23:53 +00:00
var _ filer.BucketAware = (*YdbStore)(nil)
2022-05-02 07:42:20 +00:00
func (store *YdbStore) CanDropWholeBucket() bool {
return store.SupportBucketTable
}
func (store *YdbStore) OnBucketCreation(bucket string) {
store.dbsLock.Lock()
defer store.dbsLock.Unlock()
2022-05-02 17:23:07 +00:00
if err := store.createTable(context.Background(),
2022-05-03 10:18:28 +00:00
path.Join(store.tablePathPrefix, bucket)); err != nil {
2022-05-02 07:42:20 +00:00
glog.Errorf("createTable %s: %v", bucket, err)
}
if store.dbs == nil {
return
}
store.dbs[bucket] = true
}
func (store *YdbStore) OnBucketDeletion(bucket string) {
store.dbsLock.Lock()
defer store.dbsLock.Unlock()
2022-05-02 17:23:07 +00:00
if err := store.deleteTable(context.Background(),
2022-05-03 10:18:28 +00:00
path.Join(store.tablePathPrefix, bucket)); err != nil {
2022-05-02 07:42:20 +00:00
glog.Errorf("deleteTable %s: %v", bucket, err)
}
if store.dbs == nil {
return
}
delete(store.dbs, bucket)
}
func (store *YdbStore) createTable(ctx context.Context, prefix string) error {
return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
2022-05-03 10:18:28 +00:00
return s.CreateTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE), createTableOptions()...)
2022-05-02 07:42:20 +00:00
})
}
func (store *YdbStore) deleteTable(ctx context.Context, prefix string) error {
if !store.SupportBucketTable {
2022-05-02 07:42:20 +00:00
return nil
}
2022-05-03 15:16:00 +00:00
if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
2022-05-03 10:18:28 +00:00
return s.DropTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE))
2022-05-03 15:16:00 +00:00
}); err != nil {
return err
}
glog.V(4).Infof("deleted table %s", prefix)
return nil
2022-05-02 07:42:20 +00:00
}
2022-05-03 15:16:00 +00:00
func (store *YdbStore) getPrefix(ctx context.Context, dir *string) (tablePathPrefix *string, shortDir *string) {
tablePathPrefix = &store.tablePathPrefix
shortDir = dir
2022-05-02 07:42:20 +00:00
if !store.SupportBucketTable {
return
}
prefixBuckets := store.dirBuckets + "/"
2022-05-03 15:16:00 +00:00
if strings.HasPrefix(*dir, prefixBuckets) {
// detect bucket
2022-05-03 15:16:00 +00:00
bucketAndDir := (*dir)[len(prefixBuckets):]
var bucket string
if t := strings.Index(bucketAndDir, "/"); t > 0 {
bucket = bucketAndDir[:t]
} else if t < 0 {
bucket = bucketAndDir
}
2022-05-03 15:16:00 +00:00
if bucket == "" {
2022-05-02 07:42:20 +00:00
return
}
2022-05-03 15:16:00 +00:00
2022-05-02 07:42:20 +00:00
store.dbsLock.Lock()
defer store.dbsLock.Unlock()
2022-05-03 15:16:00 +00:00
tablePathPrefixWithBucket := path.Join(store.tablePathPrefix, bucket)
2022-05-02 07:42:20 +00:00
if _, found := store.dbs[bucket]; !found {
2022-05-03 15:16:00 +00:00
if err := store.createTable(ctx, tablePathPrefixWithBucket); err == nil {
2022-05-02 07:42:20 +00:00
store.dbs[bucket] = true
2022-05-03 15:16:00 +00:00
glog.V(4).Infof("created table %s", tablePathPrefixWithBucket)
2022-05-02 07:42:20 +00:00
} else {
2022-05-03 15:16:00 +00:00
glog.Errorf("createTable %s: %v", tablePathPrefixWithBucket, err)
2022-05-02 07:42:20 +00:00
}
}
2022-05-03 15:16:00 +00:00
tablePathPrefix = &tablePathPrefixWithBucket
2022-05-02 07:42:20 +00:00
}
return
}