2018-05-26 12:32:15 +00:00
|
|
|
package abstract_sql
|
|
|
|
|
|
|
|
import (
|
2019-03-15 22:55:34 +00:00
|
|
|
"context"
|
2018-05-26 12:32:15 +00:00
|
|
|
"database/sql"
|
2018-05-27 18:52:26 +00:00
|
|
|
"fmt"
|
2020-09-01 07:21:19 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/filer"
|
2018-05-26 12:32:15 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2020-03-08 01:01:39 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
2020-01-19 20:06:19 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
2020-09-02 17:09:49 +00:00
|
|
|
"strings"
|
2018-05-26 12:32:15 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type AbstractSqlStore struct {
|
2019-12-13 08:23:05 +00:00
|
|
|
DB *sql.DB
|
|
|
|
SqlInsert string
|
|
|
|
SqlUpdate string
|
|
|
|
SqlFind string
|
|
|
|
SqlDelete string
|
|
|
|
SqlDeleteFolderChildren string
|
|
|
|
SqlListExclusive string
|
|
|
|
SqlListInclusive string
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2019-03-31 06:08:29 +00:00
|
|
|
type TxOrDB interface {
|
|
|
|
ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
|
|
|
|
QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
|
|
|
|
QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
|
2019-04-01 18:03:04 +00:00
|
|
|
tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
|
|
|
|
Isolation: sql.LevelReadCommitted,
|
|
|
|
ReadOnly: false,
|
|
|
|
})
|
2019-03-31 06:08:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return ctx, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return context.WithValue(ctx, "tx", tx), nil
|
|
|
|
}
|
|
|
|
func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
|
|
|
|
if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
|
|
|
|
return tx.Commit()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
|
|
|
|
if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
|
|
|
|
return tx.Rollback()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB {
|
|
|
|
if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
|
|
|
|
return tx
|
|
|
|
}
|
|
|
|
return store.DB
|
|
|
|
}
|
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
2018-05-26 12:32:15 +00:00
|
|
|
|
|
|
|
dir, name := entry.FullPath.DirAndName()
|
|
|
|
meta, err := entry.EncodeAttributesAndChunks()
|
|
|
|
if err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2020-09-03 18:00:20 +00:00
|
|
|
if len(entry.Chunks) > 50 {
|
|
|
|
meta = util.MaybeGzipData(meta)
|
|
|
|
}
|
|
|
|
|
2020-01-19 20:06:19 +00:00
|
|
|
res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta)
|
2018-05-26 12:32:15 +00:00
|
|
|
if err != nil {
|
2020-09-02 17:09:49 +00:00
|
|
|
if !strings.Contains(strings.ToLower(err.Error()), "duplicate") {
|
|
|
|
return fmt.Errorf("kv insert: %s", err)
|
|
|
|
}
|
2020-08-20 14:52:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// now the insert failed possibly due to duplication constraints
|
|
|
|
glog.V(1).Infof("insert %s falls back to update: %s", entry.FullPath, err)
|
|
|
|
|
|
|
|
res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("upsert %s: %s", entry.FullPath, err)
|
|
|
|
}
|
|
|
|
|
2018-05-26 12:32:15 +00:00
|
|
|
_, err = res.RowsAffected()
|
|
|
|
if err != nil {
|
2020-08-20 14:52:46 +00:00
|
|
|
return fmt.Errorf("upsert %s but no rows affected: %s", entry.FullPath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
return nil
|
2020-08-20 14:52:46 +00:00
|
|
|
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
2018-05-26 12:32:15 +00:00
|
|
|
|
|
|
|
dir, name := entry.FullPath.DirAndName()
|
|
|
|
meta, err := entry.EncodeAttributesAndChunks()
|
|
|
|
if err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
return fmt.Errorf("encode %s: %s", entry.FullPath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2020-01-19 20:06:19 +00:00
|
|
|
res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir)
|
2018-05-26 12:32:15 +00:00
|
|
|
if err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
return fmt.Errorf("update %s: %s", entry.FullPath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
_, err = res.RowsAffected()
|
|
|
|
if err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
return fmt.Errorf("update %s but no rows affected: %s", entry.FullPath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) {
|
2018-05-26 12:32:15 +00:00
|
|
|
|
|
|
|
dir, name := fullpath.DirAndName()
|
2020-01-19 20:06:19 +00:00
|
|
|
row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir)
|
2020-09-02 05:47:57 +00:00
|
|
|
|
2018-05-26 12:32:15 +00:00
|
|
|
var data []byte
|
|
|
|
if err := row.Scan(&data); err != nil {
|
2020-09-02 05:47:57 +00:00
|
|
|
if err == sql.ErrNoRows {
|
|
|
|
return nil, filer_pb.ErrNotFound
|
|
|
|
}
|
|
|
|
return nil, fmt.Errorf("find %s: %v", fullpath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
entry := &filer.Entry{
|
2018-05-26 12:32:15 +00:00
|
|
|
FullPath: fullpath,
|
|
|
|
}
|
2020-09-03 18:00:20 +00:00
|
|
|
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return entry, nil
|
|
|
|
}
|
|
|
|
|
2020-03-23 07:01:34 +00:00
|
|
|
func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
|
2018-05-26 12:32:15 +00:00
|
|
|
|
|
|
|
dir, name := fullpath.DirAndName()
|
|
|
|
|
2020-01-19 20:06:19 +00:00
|
|
|
res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir)
|
2018-05-26 12:32:15 +00:00
|
|
|
if err != nil {
|
2018-05-31 03:24:57 +00:00
|
|
|
return fmt.Errorf("delete %s: %s", fullpath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
_, err = res.RowsAffected()
|
|
|
|
if err != nil {
|
2018-05-31 03:24:57 +00:00
|
|
|
return fmt.Errorf("delete %s but no rows affected: %s", fullpath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2018-05-31 03:24:57 +00:00
|
|
|
return nil
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2020-03-23 07:01:34 +00:00
|
|
|
func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
|
2019-12-13 08:23:05 +00:00
|
|
|
|
2020-01-19 20:06:19 +00:00
|
|
|
res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(fullpath)), fullpath)
|
2019-12-13 08:23:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = res.RowsAffected()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("deleteFolderChildren %s but no rows affected: %s", fullpath, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {
|
2018-05-27 05:02:49 +00:00
|
|
|
sqlText := store.SqlListExclusive
|
2018-05-26 20:35:56 +00:00
|
|
|
if inclusive {
|
2018-05-27 05:02:49 +00:00
|
|
|
sqlText = store.SqlListInclusive
|
2018-05-26 20:35:56 +00:00
|
|
|
}
|
|
|
|
|
2020-08-05 19:37:42 +00:00
|
|
|
rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit)
|
2018-05-26 12:32:15 +00:00
|
|
|
if err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
return nil, fmt.Errorf("list %s : %v", fullpath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
defer rows.Close()
|
|
|
|
|
|
|
|
for rows.Next() {
|
|
|
|
var name string
|
|
|
|
var data []byte
|
|
|
|
if err = rows.Scan(&name, &data); err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
glog.V(0).Infof("scan %s : %v", fullpath, err)
|
|
|
|
return nil, fmt.Errorf("scan %s: %v", fullpath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
entry := &filer.Entry{
|
2020-03-23 07:01:34 +00:00
|
|
|
FullPath: util.NewFullPath(string(fullpath), name),
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
2020-09-03 18:00:20 +00:00
|
|
|
if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
|
2018-05-27 05:02:49 +00:00
|
|
|
glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err)
|
|
|
|
return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err)
|
2018-05-26 12:32:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
entries = append(entries, entry)
|
|
|
|
}
|
|
|
|
|
|
|
|
return entries, nil
|
|
|
|
}
|
2020-03-15 03:30:26 +00:00
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) {
|
2020-08-05 19:38:59 +00:00
|
|
|
return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
|
2020-08-05 19:37:42 +00:00
|
|
|
}
|
2020-03-15 03:30:26 +00:00
|
|
|
|
|
|
|
func (store *AbstractSqlStore) Shutdown() {
|
|
|
|
store.DB.Close()
|
|
|
|
}
|