From 33a9e5e2d1948d08ec6456be6df3a6683a780905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 22:19:16 +0500 Subject: [PATCH 01/38] test ListDirectoryPrefixedEntries --- .../filer2/abstract_sql/abstract_sql_store.go | 69 ++++++++++++++++++- weed/filer2/cassandra/cassandra_store.go | 31 +++++++++ weed/filer2/etcd/etcd_store.go | 34 ++++++++- weed/filer2/filer.go | 10 +-- weed/filer2/filerstore.go | 5 +- weed/filer2/leveldb/leveldb_store.go | 31 +++++++++ weed/filer2/leveldb2/leveldb2_store.go | 31 +++++++++ weed/filer2/mongodb/mongodb_store.go | 31 +++++++++ weed/filer2/mysql/mysql_store.go | 2 +- weed/filer2/redis/universal_redis_store.go | 30 ++++++++ weed/filer2/redis2/universal_redis_store.go | 31 +++++++++ weed/server/filer_grpc_server.go | 9 +-- 12 files changed, 294 insertions(+), 20 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index 5ade18960..ed0d6e8ef 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -150,8 +150,75 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { +//func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +// sqlText := store.SqlListExclusive +// if inclusive { +// sqlText = store.SqlListInclusive +// } +// +// rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit) +// if err != nil { +// return nil, fmt.Errorf("list %s : %v", fullpath, err) +// } +// defer rows.Close() +// +// for rows.Next() { +// var name string +// var data []byte +// if err = rows.Scan(&name, &data); err != nil { +// glog.V(0).Infof("scan %s : %v", fullpath, err) +// return nil, fmt.Errorf("scan %s: %v", fullpath, err) +// } +// +// entry := &filer2.Entry{ +// FullPath: util.NewFullPath(string(fullpath), name), +// } +// if err = entry.DecodeAttributesAndChunks(data); err != nil { +// glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) +// return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) +// } +// +// entries = append(entries, entry) +// } +// +// return entries, nil +//} +//func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +// return nil, fmt.Errorf("not implemented") +// +//} +func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { sqlText := store.SqlListExclusive if inclusive { sqlText = store.SqlListInclusive diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 5dd7d8036..225ad02a3 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -3,6 +3,7 @@ package cassandra import ( "context" "fmt" + "strings" "github.com/gocql/gocql" @@ -126,6 +127,36 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath return nil } +func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 2ef65b4a0..5fbdb60aa 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -135,9 +135,37 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryEntries( - ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, -) (entries []*filer2.Entry, err error) { +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + +func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, "") resp, err := store.client.Get(ctx, string(directoryPrefix), diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index dd4c38857..92764f600 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -259,15 +259,15 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } -func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) { +func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) ([]*Entry, error) { if strings.HasSuffix(string(p), "/") && len(p) > 1 { p = p[0 : len(p)-1] } var makeupEntries []*Entry - entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit) + entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) for expiredCount > 0 && err == nil { - makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount) + makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix) if err == nil { entries = append(entries, makeupEntries...) } @@ -276,8 +276,8 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start return entries, err } -func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int) (entries []*Entry, expiredCount int, lastFileName string, err error) { - listedEntries, listErr := f.Store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit) +func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) { + listedEntries, listErr := f.Store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) if listErr != nil { return listedEntries, expiredCount, "", listErr } diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 7c518c6fe..3df43e6b2 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -21,6 +21,7 @@ type FilerStore interface { DeleteEntry(context.Context, util.FullPath) (err error) DeleteFolderChildren(context.Context, util.FullPath) (err error) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) + ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error @@ -112,14 +113,14 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. return fsw.ActualStore.DeleteFolderChildren(ctx, fp) } -func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) { +func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc() start := time.Now() defer func() { stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() - entries, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) if err != nil { return nil, err } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 31919ca49..4b42340e0 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "strings" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" @@ -159,6 +160,36 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index c907e8746..46b2558cb 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "strings" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" @@ -168,6 +169,36 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 375a457a4..a1502430c 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -11,6 +11,7 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/x/bsonx" + "strings" "time" ) @@ -167,6 +168,36 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut return nil } +func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { var where = bson.M{"directory": string(fullpath), "name": bson.M{"$gt": startFileName}} diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index 63d99cd9d..b22ac19fb 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -41,7 +41,7 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int, interpolateParams bool) (err error) { - + //AND name like CONCAT(?,'%') store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index e5b9e8840..0c90b8993 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -121,6 +121,36 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full return nil } +func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 420336b46..483dd621f 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -3,6 +3,7 @@ package redis2 import ( "context" "fmt" + "strings" "time" "github.com/go-redis/redis" @@ -116,6 +117,36 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } +func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 48e9253f0..eaace2fc2 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "strconv" - "strings" "time" "github.com/chrislusf/seaweedfs/weed/filer2" @@ -59,7 +58,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file lastFileName := req.StartFromFileName includeLastFile := req.InclusiveStartFrom for limit > 0 { - entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit) + entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit, req.Prefix) if err != nil { return err @@ -74,12 +73,6 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file lastFileName = entry.Name() - if req.Prefix != "" { - if !strings.HasPrefix(entry.Name(), req.Prefix) { - continue - } - } - if err := stream.Send(&filer_pb.ListEntriesResponse{ Entry: &filer_pb.Entry{ Name: entry.Name(), From dc9fc0125419028da895c01f754305891fce2eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 23:38:00 +0500 Subject: [PATCH 02/38] test ListDirectoryPrefixedEntries --- .../filer2/abstract_sql/abstract_sql_store.go | 1 + weed/filer2/etcd/etcd_store.go | 2 +- weed/filer2/filer.go | 2 +- weed/filer2/filer_buckets.go | 2 +- weed/filer2/filer_delete_entry.go | 2 +- weed/filer2/filer_notify.go | 4 ++-- weed/filer2/filerstore.go | 21 +++++++++++++++++-- weed/filer2/leveldb/leveldb_store.go | 2 +- weed/filer2/leveldb2/leveldb2_store.go | 2 +- 9 files changed, 28 insertions(+), 10 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index ed0d6e8ef..b6da8e74e 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 5fbdb60aa..a1bf9bf3c 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -135,7 +135,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { count := 0 notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) if err != nil { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 92764f600..cc8fecc42 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -277,7 +277,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start } func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) { - listedEntries, listErr := f.Store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) + listedEntries, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) if listErr != nil { return listedEntries, expiredCount, "", listErr } diff --git a/weed/filer2/filer_buckets.go b/weed/filer2/filer_buckets.go index 7a57e7ee1..6b7c2c31a 100644 --- a/weed/filer2/filer_buckets.go +++ b/weed/filer2/filer_buckets.go @@ -29,7 +29,7 @@ func (f *Filer) LoadBuckets() { limit := math.MaxInt32 - entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit) + entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "") if err != nil { glog.V(1).Infof("no buckets found: %v", err) diff --git a/weed/filer2/filer_delete_entry.go b/weed/filer2/filer_delete_entry.go index 35099a472..17624bbd6 100644 --- a/weed/filer2/filer_delete_entry.go +++ b/weed/filer2/filer_delete_entry.go @@ -58,7 +58,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry lastFileName := "" includeLastFile := false for { - entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize) + entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "") if err != nil { glog.Errorf("list folder %s: %v", entry.FullPath, err) return nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go index e5f9eba0a..97ae8e9c8 100644 --- a/weed/filer2/filer_notify.go +++ b/weed/filer2/filer_notify.go @@ -96,13 +96,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func( sizeBuf := make([]byte, 4) startTsNs := startTime.UnixNano() - dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366) + dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "") if listDayErr != nil { return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr) } for _, dayEntry := range dayEntries { // println("checking day", dayEntry.FullPath) - hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60) + hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "") if listHourMinuteErr != nil { return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) } diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 3df43e6b2..8edbe3034 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -113,7 +113,24 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. return fsw.ActualStore.DeleteFolderChildren(ctx, fp) } -func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { +func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) { + stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc() + start := time.Now() + defer func() { + stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) + }() + + entries, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + for _, entry := range entries { + filer_pb.AfterEntryDeserialization(entry.Chunks) + } + return entries, err +} + +func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc() start := time.Now() defer func() { @@ -127,7 +144,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath for _, entry := range entries { filer_pb.AfterEntryDeserialization(entry.Chunks) } - return entries, err + return entries, nil } func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) { diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 4b42340e0..5b118884e 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -160,7 +160,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { count := 0 notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) if err != nil { diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 46b2558cb..fdff3c92d 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -169,7 +169,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { count := 0 notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) if err != nil { From 67f134ebd7c0a84e06f36979886afdd7e3435eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 23:44:48 +0500 Subject: [PATCH 03/38] test ListDirectoryPrefixedEntries --- weed/filer2/leveldb/leveldb_store_test.go | 6 +++--- weed/filer2/leveldb2/leveldb2_store_test.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/weed/filer2/leveldb/leveldb_store_test.go b/weed/filer2/leveldb/leveldb_store_test.go index 77df07a9b..09dde72d4 100644 --- a/weed/filer2/leveldb/leveldb_store_test.go +++ b/weed/filer2/leveldb/leveldb_store_test.go @@ -50,14 +50,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100) + entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filer2/leveldb2/leveldb2_store_test.go b/weed/filer2/leveldb2/leveldb2_store_test.go index b211d86e4..d212d24ba 100644 --- a/weed/filer2/leveldb2/leveldb2_store_test.go +++ b/weed/filer2/leveldb2/leveldb2_store_test.go @@ -50,14 +50,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100) + entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if err != nil { t.Errorf("list entries: %v", err) return From 2d3b355fb6a5060ec32583478598a13344c90f74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 23:56:06 +0500 Subject: [PATCH 04/38] test ListDirectoryPrefixedEntries --- weed/server/filer_grpc_server_rename.go | 2 +- weed/server/filer_server_handlers_read_dir.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index 9642fec24..219067c86 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -70,7 +70,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. includeLastFile := false for { - entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024) + entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "") if err != nil { return err } diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index ae28fc1db..9ca0209f4 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -32,7 +32,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque lastFileName := r.FormValue("lastFileName") - entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit) + entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, "") if err != nil { glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err) From a457c308ad31cfa90919fb9f430aeb9faa3b06d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 6 Aug 2020 00:24:31 +0500 Subject: [PATCH 05/38] test ListDirectoryPrefixedEntries --- weed/filer2/abstract_sql/abstract_sql_store.go | 8 +++++++- weed/filer2/cassandra/cassandra_store.go | 8 +++++++- weed/filer2/etcd/etcd_store.go | 8 +++++++- weed/filer2/leveldb/leveldb_store.go | 8 +++++++- weed/filer2/leveldb2/leveldb2_store.go | 8 +++++++- weed/filer2/mongodb/mongodb_store.go | 8 +++++++- weed/filer2/redis/universal_redis_store.go | 8 +++++++- weed/filer2/redis2/universal_redis_store.go | 8 +++++++- 8 files changed, 56 insertions(+), 8 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index b6da8e74e..a90c1341f 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -199,8 +199,10 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -210,10 +212,14 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 225ad02a3..dc29371d8 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -137,8 +137,10 @@ func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, f if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -148,10 +150,14 @@ func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, f break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index a1bf9bf3c..9b4e21917 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -145,8 +145,10 @@ func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpa if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -156,10 +158,14 @@ func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpa break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 5b118884e..0d6a62212 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -170,8 +170,10 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -181,10 +183,14 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index fdff3c92d..a8e915e79 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -179,8 +179,10 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -190,10 +192,14 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index a1502430c..00f710f1f 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -178,8 +178,10 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, ful if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -189,10 +191,14 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, ful break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 0c90b8993..a3a8f866e 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -131,8 +131,10 @@ func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Conte if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -142,10 +144,14 @@ func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Conte break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 483dd621f..289e85545 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -127,8 +127,10 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -138,10 +140,14 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil From b231f7bdab084683099f429631e4d1dcfb6ae7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 6 Aug 2020 00:37:42 +0500 Subject: [PATCH 06/38] ListDirectoryPrefixedEntries --- .../filer2/abstract_sql/abstract_sql_store.go | 82 ++----------------- weed/filer2/mysql/mysql_store.go | 6 +- weed/filer2/postgres/postgres_store.go | 4 +- 3 files changed, 10 insertions(+), 82 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index a90c1341f..fb0cadee5 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -4,8 +4,6 @@ import ( "context" "database/sql" "fmt" - "strings" - "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -151,87 +149,13 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -//func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { -// sqlText := store.SqlListExclusive -// if inclusive { -// sqlText = store.SqlListInclusive -// } -// -// rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit) -// if err != nil { -// return nil, fmt.Errorf("list %s : %v", fullpath, err) -// } -// defer rows.Close() -// -// for rows.Next() { -// var name string -// var data []byte -// if err = rows.Scan(&name, &data); err != nil { -// glog.V(0).Infof("scan %s : %v", fullpath, err) -// return nil, fmt.Errorf("scan %s: %v", fullpath, err) -// } -// -// entry := &filer2.Entry{ -// FullPath: util.NewFullPath(string(fullpath), name), -// } -// if err = entry.DecodeAttributesAndChunks(data); err != nil { -// glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) -// return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) -// } -// -// entries = append(entries, entry) -// } -// -// return entries, nil -//} -//func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { -// return nil, fmt.Errorf("not implemented") -// -//} - func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil -} - -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { sqlText := store.SqlListExclusive if inclusive { sqlText = store.SqlListInclusive } - rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), limit) + rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit) if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -258,6 +182,10 @@ func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpat return entries, nil } +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + return nil, fmt.Errorf("not implemented") + +} func (store *AbstractSqlStore) Shutdown() { store.DB.Close() diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index b22ac19fb..9f748445e 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -41,14 +41,14 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int, interpolateParams bool) (err error) { - //AND name like CONCAT(?,'%') + // store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?" store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=? AND directory=?" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? ORDER BY NAME ASC LIMIT ?" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? ORDER BY NAME ASC LIMIT ?" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like CONCAT(?,'%') ORDER BY NAME ASC LIMIT ?" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like CONCAT(?,'%') ORDER BY NAME ASC LIMIT ?" sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) if interpolateParams { diff --git a/weed/filer2/postgres/postgres_store.go b/weed/filer2/postgres/postgres_store.go index 51c069aae..87eb6aca2 100644 --- a/weed/filer2/postgres/postgres_store.go +++ b/weed/filer2/postgres/postgres_store.go @@ -46,8 +46,8 @@ func (store *PostgresStore) initialize(user, password, hostname string, port int store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=$1 AND directory=$2" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like CONCAT($4,'%')ORDER BY NAME ASC LIMIT $5" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like CONCAT($4,'%') ORDER BY NAME ASC LIMIT $5" sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, user, sslmode) if password != "" { From 9ca011e3cab51f7f0489df463ba4566216d8cf8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 6 Aug 2020 00:38:59 +0500 Subject: [PATCH 07/38] ListDirectoryPrefixedEntries --- weed/filer2/abstract_sql/abstract_sql_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index fb0cadee5..d1ce83de6 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -182,8 +182,8 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("not implemented") +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { + return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") } From 2ea638f865c9bebe89c05d8a4acf90a1447375f3 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 17:13:56 +0500 Subject: [PATCH 08/38] filer store wrapper can implement the logic to filter by prefi --- .../filer2/abstract_sql/abstract_sql_store.go | 2 +- weed/filer2/cassandra/cassandra_store.go | 36 +-------------- weed/filer2/etcd/etcd_store.go | 34 +------------- weed/filer2/filerstore.go | 44 ++++++++++++++++++- weed/filer2/leveldb/leveldb_store.go | 36 +-------------- weed/filer2/leveldb2/leveldb2_store.go | 40 ++--------------- weed/filer2/mongodb/mongodb_store.go | 35 +-------------- weed/filer2/redis/universal_redis_store.go | 34 +------------- weed/filer2/redis2/universal_redis_store.go | 35 +-------------- 9 files changed, 53 insertions(+), 243 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index d1ce83de6..7f591413a 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -182,9 +182,9 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } + func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") - } func (store *AbstractSqlStore) Shutdown() { diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index dc29371d8..a996a6401 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -3,8 +3,6 @@ package cassandra import ( "context" "fmt" - "strings" - "github.com/gocql/gocql" "github.com/chrislusf/seaweedfs/weed/filer2" @@ -128,39 +126,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 9b4e21917..043bf995d 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -136,39 +136,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 8edbe3034..8d4e75e9a 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -2,6 +2,8 @@ package filer2 import ( "context" + "fmt" + "strings" "time" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -22,6 +24,7 @@ type FilerStore interface { DeleteFolderChildren(context.Context, util.FullPath) (err error) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) + ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error @@ -136,8 +139,10 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, defer func() { stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() - entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + if err == fmt.Errorf("UNSUPPORTED") { + entries, err = fsw.ListDirectoryUnSupPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + } if err != nil { return nil, err } @@ -147,6 +152,43 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } +func (fsw *FilerStoreWrapper) ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { + count := 0 + notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + + var lastFileName string + for count < limit { + for _, entry := range notPrefixed { + lastFileName = entry.Name() + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + + if len(notPrefixed) == 0 { + break + } + } + + return entries, nil +} + func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) { return fsw.ActualStore.BeginTransaction(ctx) } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 0d6a62212..104e3d026 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -4,8 +4,6 @@ import ( "bytes" "context" "fmt" - "strings" - "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" @@ -161,39 +159,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index a8e915e79..478661d0e 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -5,14 +5,12 @@ import ( "context" "crypto/md5" "fmt" - "io" - "os" - "strings" - "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + "io" + "os" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" @@ -170,39 +168,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 00f710f1f..1fa18f872 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -11,7 +11,6 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/x/bsonx" - "strings" "time" ) @@ -169,39 +168,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index a3a8f866e..9c71b045d 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -122,39 +122,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 289e85545..6490f7bed 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -3,7 +3,6 @@ package redis2 import ( "context" "fmt" - "strings" "time" "github.com/go-redis/redis" @@ -118,39 +117,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From 82ea121d091e2b498b48e19ee8f7f5e7f48f15e2 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 18:50:06 +0500 Subject: [PATCH 09/38] rm func --- weed/filer2/filerstore.go | 71 ++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 8d4e75e9a..9ea39cf11 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -2,7 +2,6 @@ package filer2 import ( "context" - "fmt" "strings" "time" @@ -24,7 +23,6 @@ type FilerStore interface { DeleteFolderChildren(context.Context, util.FullPath) (err error) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) - ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error @@ -140,10 +138,39 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err == fmt.Errorf("UNSUPPORTED") { - entries, err = fsw.ListDirectoryUnSupPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - } - if err != nil { + if err.Error() == "UNSUPPORTED" { + count := 0 + notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + if prefix == "" { + entries = notPrefixed + } else { + var lastFileName string + for count < limit { + for _, entry := range notPrefixed { + lastFileName = entry.Name() + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + + if len(notPrefixed) == 0 { + break + } + } + } + } else if err != nil { return nil, err } for _, entry := range entries { @@ -153,38 +180,6 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, } func (fsw *FilerStoreWrapper) ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { - count := 0 - notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } return entries, nil } From 60a86cfe0496497ae4942d3e8925159154faab54 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:35:16 +0500 Subject: [PATCH 10/38] add const --- weed/filer2/cassandra/cassandra_store.go | 3 ++- weed/filer2/etcd/etcd_store.go | 3 ++- weed/filer2/filer.go | 5 ++++- weed/filer2/filerstore.go | 7 +------ weed/filer2/leveldb/leveldb_store.go | 7 ++++--- weed/filer2/leveldb2/leveldb2_store.go | 7 ++++--- weed/filer2/mongodb/mongodb_store.go | 3 ++- weed/filer2/redis/universal_redis_store.go | 3 ++- weed/filer2/redis2/universal_redis_store.go | 3 ++- 9 files changed, 23 insertions(+), 18 deletions(-) diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index a996a6401..8214935f9 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -2,6 +2,7 @@ package cassandra import ( "context" + "errors" "fmt" "github.com/gocql/gocql" @@ -126,7 +127,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 043bf995d..93f3f3781 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -2,6 +2,7 @@ package etcd import ( "context" + "errors" "fmt" "strings" "time" @@ -136,7 +137,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index cc8fecc42..ef81cae2b 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -18,7 +18,10 @@ import ( "github.com/chrislusf/seaweedfs/weed/wdclient" ) -const PaginationSize = 1024 * 256 +const ( + PaginationSize = 1024 * 256 + UnsupportedListDirectoryPrefixedErr = "UNSUPPORTED" +) var ( OS_UID = uint32(os.Getuid()) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 9ea39cf11..3c1e555b1 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == "UNSUPPORTED" { + if err.Error() == UnsupportedListDirectoryPrefixedErr { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { @@ -179,11 +179,6 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } -func (fsw *FilerStoreWrapper) ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { - - return entries, nil -} - func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) { return fsw.ActualStore.BeginTransaction(ctx) } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 104e3d026..787671c55 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -3,9 +3,10 @@ package leveldb import ( "bytes" "context" + "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" + leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" @@ -49,7 +50,7 @@ func (store *LevelDBStore) initialize(dir string) (err error) { } if store.db, err = leveldb.OpenFile(dir, opts); err != nil { - if errors.IsCorrupted(err) { + if leveldb_errors.IsCorrupted(err) { store.db, err = leveldb.RecoverFile(dir, opts) } if err != nil { @@ -159,7 +160,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 478661d0e..4fc110d69 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -4,9 +4,10 @@ import ( "bytes" "context" "crypto/md5" + "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" + leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" "io" @@ -52,7 +53,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { dbFolder := fmt.Sprintf("%s/%02d", dir, d) os.MkdirAll(dbFolder, 0755) db, dbErr := leveldb.OpenFile(dbFolder, opts) - if errors.IsCorrupted(dbErr) { + if leveldb_errors.IsCorrupted(dbErr) { db, dbErr = leveldb.RecoverFile(dbFolder, opts) } if dbErr != nil { @@ -168,7 +169,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 1fa18f872..314f36ce2 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -2,6 +2,7 @@ package mongodb import ( "context" + "errors" "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" @@ -168,7 +169,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 9c71b045d..66450b5a7 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -2,6 +2,7 @@ package redis import ( "context" + "errors" "fmt" "sort" "strings" @@ -122,7 +123,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 6490f7bed..9878799cf 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -2,6 +2,7 @@ package redis2 import ( "context" + "errors" "fmt" "time" @@ -117,7 +118,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From 7f013d77d6f42b5116d59703520f2e87a6ae2540 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:43:32 +0500 Subject: [PATCH 11/38] add var UnsupportedListDirectoryPrefixedErr --- weed/filer2/cassandra/cassandra_store.go | 3 +-- weed/filer2/etcd/etcd_store.go | 3 +-- weed/filer2/filer.go | 9 +++++---- weed/filer2/filerstore.go | 2 +- weed/filer2/leveldb/leveldb_store.go | 3 +-- weed/filer2/leveldb2/leveldb2_store.go | 3 +-- weed/filer2/mongodb/mongodb_store.go | 3 +-- weed/filer2/redis/universal_redis_store.go | 3 +-- weed/filer2/redis2/universal_redis_store.go | 3 +-- 9 files changed, 13 insertions(+), 19 deletions(-) diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 8214935f9..40f371e2f 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -2,7 +2,6 @@ package cassandra import ( "context" - "errors" "fmt" "github.com/gocql/gocql" @@ -127,7 +126,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 93f3f3781..d0b1ff202 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -2,7 +2,6 @@ package etcd import ( "context" - "errors" "fmt" "strings" "time" @@ -137,7 +136,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index ef81cae2b..a94052996 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -2,6 +2,7 @@ package filer2 import ( "context" + "errors" "fmt" "os" "strings" @@ -19,13 +20,13 @@ import ( ) const ( - PaginationSize = 1024 * 256 - UnsupportedListDirectoryPrefixedErr = "UNSUPPORTED" + PaginationSize = 1024 * 256 ) var ( - OS_UID = uint32(os.Getuid()) - OS_GID = uint32(os.Getgid()) + OS_UID = uint32(os.Getuid()) + OS_GID = uint32(os.Getgid()) + UnsupportedListDirectoryPrefixedErr = errors.New("UNSUPPORTED") ) type Filer struct { diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 3c1e555b1..642ebb3c8 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == UnsupportedListDirectoryPrefixedErr { + if err.Error() == UnsupportedListDirectoryPrefixedErr.Error() { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 787671c55..2ea88635b 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -3,7 +3,6 @@ package leveldb import ( "bytes" "context" - "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" @@ -160,7 +159,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 4fc110d69..2a9c2fe3f 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/md5" - "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" @@ -169,7 +168,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 314f36ce2..080d6248f 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -2,7 +2,6 @@ package mongodb import ( "context" - "errors" "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" @@ -169,7 +168,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 66450b5a7..8cc70c72b 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -2,7 +2,6 @@ package redis import ( "context" - "errors" "fmt" "sort" "strings" @@ -123,7 +122,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 9878799cf..2ce7ebec7 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -2,7 +2,6 @@ package redis2 import ( "context" - "errors" "fmt" "time" @@ -118,7 +117,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From a8de332beb933db2acc1c93b7c352fb7f2c20647 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:45:44 +0500 Subject: [PATCH 12/38] fix --- weed/filer2/filer.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index a94052996..603e51fa2 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -19,9 +19,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/wdclient" ) -const ( - PaginationSize = 1024 * 256 -) +const PaginationSize = 1024 * 256 var ( OS_UID = uint32(os.Getuid()) From 22f32e75c411fbf13c209ebaeadb673d72a9a29b Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:52:05 +0500 Subject: [PATCH 13/38] rename --- weed/filer2/cassandra/cassandra_store.go | 2 +- weed/filer2/etcd/etcd_store.go | 2 +- weed/filer2/filer.go | 2 +- weed/filer2/filerstore.go | 2 +- weed/filer2/leveldb/leveldb_store.go | 2 +- weed/filer2/leveldb2/leveldb2_store.go | 2 +- weed/filer2/mongodb/mongodb_store.go | 2 +- weed/filer2/redis/universal_redis_store.go | 2 +- weed/filer2/redis2/universal_redis_store.go | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 40f371e2f..4d845c2fa 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -126,7 +126,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index d0b1ff202..6f4c3ce5c 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -136,7 +136,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 603e51fa2..9d485ad8a 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -24,7 +24,7 @@ const PaginationSize = 1024 * 256 var ( OS_UID = uint32(os.Getuid()) OS_GID = uint32(os.Getgid()) - UnsupportedListDirectoryPrefixedErr = errors.New("UNSUPPORTED") + ErrUnsupportedListDirectoryPrefixed = errors.New("UNSUPPORTED") ) type Filer struct { diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 642ebb3c8..b55aeb4d6 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == UnsupportedListDirectoryPrefixedErr.Error() { + if err.Error() == ErrUnsupportedListDirectoryPrefixed.Error() { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 2ea88635b..1c08d2831 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -159,7 +159,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 2a9c2fe3f..ca9d6f04d 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -168,7 +168,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 080d6248f..661aa4ea0 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -168,7 +168,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 8cc70c72b..fc2abef6c 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -122,7 +122,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 2ce7ebec7..c639635ef 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -117,7 +117,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From 18b98f9747082c9cd5e7ee83f98ab3d6c0eeddb3 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:55:18 +0500 Subject: [PATCH 14/38] not convert err to string --- weed/filer2/filerstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index b55aeb4d6..db53734bc 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == ErrUnsupportedListDirectoryPrefixed.Error() { + if err == ErrUnsupportedListDirectoryPrefixed { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { From 9a195bebfd6c803161d07ca80b227dd058719aa5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 22:13:13 +0500 Subject: [PATCH 15/38] accurate limit --- weed/filer2/filerstore.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index db53734bc..32c4c94fb 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -155,12 +155,11 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, count++ entries = append(entries, entry) } + if count >= limit { + goto Exit + } } - if count >= limit { - break - } - - notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit) if err != nil { return nil, err } @@ -169,6 +168,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, break } } + Exit: } } else if err != nil { return nil, err From f0c89cfacdc6d83ea4a3ce6241d593506529b6c1 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 31 Aug 2020 10:23:31 -0700 Subject: [PATCH 16/38] go fmt --- weed/server/master_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/master_server.go b/weed/server/master_server.go index ae59636ad..657b170c2 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -209,7 +209,7 @@ func (ms *MasterServer) startAdminScripts() { scriptLines = append(scriptLines, "unlock") } - masterAddress := fmt.Sprintf("%s:%d",ms.option.Host, ms.option.Port) + masterAddress := fmt.Sprintf("%s:%d", ms.option.Host, ms.option.Port) var shellOptions shell.ShellOptions shellOptions.GrpcDialOption = security.LoadClientTLS(v, "grpc.master") From 8a1c8e41b301fff49f0e55a4810c056406f2bf7d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 31 Aug 2020 10:39:24 -0700 Subject: [PATCH 17/38] simplify if else logic --- weed/filer2/filerstore.go | 67 ++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index ff35ae3f3..1d677e507 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -145,38 +145,9 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) if err == ErrUnsupportedListDirectoryPrefixed { - count := 0 - notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) - if err != nil { - return nil, err - } - if prefix == "" { - entries = notPrefixed - } else { - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - if count >= limit { - goto Exit - } - } - notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - Exit: - } - } else if err != nil { + entries, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + } + if err != nil { return nil, err } for _, entry := range entries { @@ -185,6 +156,38 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } +func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { + entries, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return + } + + count := 0 + var lastFileName string + notPrefixed := entries + entries = nil + for count < limit && len(notPrefixed) > 0 { + for _, entry := range notPrefixed { + lastFileName = entry.Name() + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count < limit { + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit-count) + if err != nil { + return + } + } + } + return +} + func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) { return fsw.ActualStore.BeginTransaction(ctx) } From c8398bdb09fceff65d7d023c6ee9b9a85148d31e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 31 Aug 2020 10:41:05 -0700 Subject: [PATCH 18/38] adjust metrics --- weed/filer2/filerstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 1d677e507..f368e119d 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,10 +138,10 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath } func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { - stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc() + stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "prefixList").Inc() start := time.Now() defer func() { - stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) + stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "prefixList").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) if err == ErrUnsupportedListDirectoryPrefixed { From 97733c3dff54ddbc94e8e3246cc5824a9d77f096 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 31 Aug 2020 10:49:17 -0700 Subject: [PATCH 19/38] typo --- weed/server/master_grpc_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index d310a27d4..108892f92 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -136,7 +136,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ } if len(heartbeat.EcShards) > 0 || heartbeat.HasNoEcShards { - glog.V(1).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards) + glog.V(1).Infof("master received ec shards from %s: %+v", dn.Url(), heartbeat.EcShards) newShards, deletedShards := ms.Topo.SyncDataNodeEcShards(heartbeat.EcShards, dn) // broadcast the ec vid changes to master clients From dd176cdb8bf5a25dcde3bd58a0fa6a792d7d5a1b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 31 Aug 2020 11:28:03 -0700 Subject: [PATCH 20/38] avoid tiny step pagination --- weed/filer2/filerstore.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index f368e119d..dd5abe421 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -176,10 +176,13 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) + if count >= limit { + break + } } } if count < limit { - notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit-count) + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit) if err != nil { return } From dbb10e0f0b7e9d9341ca170ce2bd50836bbca4f8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 31 Aug 2020 17:59:29 -0700 Subject: [PATCH 21/38] testing with more than 30days --- weed/storage/needle/volume_ttl_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/weed/storage/needle/volume_ttl_test.go b/weed/storage/needle/volume_ttl_test.go index 0afebebf5..f75453593 100644 --- a/weed/storage/needle/volume_ttl_test.go +++ b/weed/storage/needle/volume_ttl_test.go @@ -30,6 +30,11 @@ func TestTTLReadWrite(t *testing.T) { t.Errorf("5d ttl:%v", ttl) } + ttl, _ = ReadTTL("50d") + if ttl.Minutes() != 50*24*60 { + t.Errorf("50d ttl:%v", ttl) + } + ttl, _ = ReadTTL("5w") if ttl.Minutes() != 5*7*24*60 { t.Errorf("5w ttl:%v", ttl) From 38e06d783d0a910c3df8e22bd097d3409e5d5312 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 31 Aug 2020 18:10:53 -0700 Subject: [PATCH 22/38] volume: check disk space before compaction fix https://github.com/chrislusf/seaweedfs/issues/1440 --- weed/storage/store_vacuum.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/weed/storage/store_vacuum.go b/weed/storage/store_vacuum.go index 38159496e..32666a417 100644 --- a/weed/storage/store_vacuum.go +++ b/weed/storage/store_vacuum.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -16,6 +17,10 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) { } func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64) error { if v := s.findVolume(vid); v != nil { + s := stats.NewDiskStatus(v.dir) + if int64(s.Free) < preallocate { + return fmt.Errorf("free space: %d bytes, not enough for %d bytes", s.Free, preallocate) + } return v.Compact2(preallocate, compactionBytePerSecond) } return fmt.Errorf("volume id %d is not found during compact", vid) From eb7929a9714d5d4ea8d9d70f58198b09bc459ead Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 00:21:19 -0700 Subject: [PATCH 23/38] rename filer2 to filer --- unmaintained/see_log_entry/see_log_entry.go | 4 +-- .../abstract_sql/abstract_sql_store.go | 16 ++++----- weed/{filer2 => filer}/cassandra/README.txt | 0 .../cassandra/cassandra_store.go | 20 +++++------ weed/{filer2 => filer}/configuration.go | 2 +- weed/{filer2 => filer}/entry.go | 2 +- weed/{filer2 => filer}/entry_codec.go | 2 +- weed/{filer2 => filer}/etcd/etcd_store.go | 20 +++++------ weed/{filer2 => filer}/filechunk_manifest.go | 2 +- .../filechunk_manifest_test.go | 2 +- weed/{filer2 => filer}/filechunks.go | 2 +- weed/{filer2 => filer}/filechunks2_test.go | 2 +- weed/{filer2 => filer}/filechunks_test.go | 2 +- weed/{filer2 => filer}/filer.go | 2 +- weed/{filer2 => filer}/filer_buckets.go | 2 +- weed/{filer2 => filer}/filer_delete_entry.go | 2 +- weed/{filer2 => filer}/filer_deletion.go | 2 +- weed/{filer2 => filer}/filer_notify.go | 2 +- weed/{filer2 => filer}/filer_notify_append.go | 2 +- weed/{filer2 => filer}/filer_notify_test.go | 2 +- weed/{filer2 => filer}/filerstore.go | 2 +- .../leveldb/leveldb_store.go | 20 +++++------ .../leveldb/leveldb_store_test.go | 10 +++--- .../leveldb2/leveldb2_local_store.go | 4 +-- .../leveldb2/leveldb2_store.go | 20 +++++------ .../leveldb2/leveldb2_store_test.go | 10 +++--- weed/{filer2 => filer}/meta_aggregator.go | 2 +- weed/{filer2 => filer}/meta_replay.go | 2 +- .../mongodb/mongodb_store.go | 20 +++++------ weed/{filer2 => filer}/mysql/mysql_store.go | 6 ++-- weed/{filer2 => filer}/permission.go | 2 +- weed/{filer2 => filer}/postgres/README.txt | 0 .../postgres/postgres_store.go | 6 ++-- weed/{filer2 => filer}/reader_at.go | 2 +- weed/{filer2 => filer}/reader_at_test.go | 2 +- .../redis/redis_cluster_store.go | 4 +-- weed/{filer2 => filer}/redis/redis_store.go | 4 +-- .../redis/universal_redis_store.go | 16 ++++----- .../redis2/redis_cluster_store.go | 4 +-- weed/{filer2 => filer}/redis2/redis_store.go | 4 +-- .../redis2/universal_redis_store.go | 16 ++++----- weed/{filer2 => filer}/stream.go | 2 +- weed/{filer2 => filer}/topics.go | 2 +- weed/filesys/dir.go | 8 ++--- weed/filesys/dir_link.go | 4 +-- weed/filesys/file.go | 14 ++++---- weed/filesys/filehandle.go | 20 +++++------ weed/filesys/meta_cache/meta_cache.go | 20 +++++------ weed/filesys/meta_cache/meta_cache_init.go | 4 +-- .../meta_cache/meta_cache_subscribe.go | 6 ++-- weed/filesys/wfs_deletion.go | 6 ++-- weed/filesys/wfs_write.go | 4 +-- weed/messaging/broker/broker_grpc_server.go | 6 ++-- .../broker/broker_grpc_server_publish.go | 4 +-- .../broker/broker_grpc_server_subscribe.go | 6 ++-- weed/messaging/broker/topic_manager.go | 4 +-- weed/replication/sink/azuresink/azure_sink.go | 6 ++-- weed/replication/sink/b2sink/b2_sink.go | 6 ++-- weed/replication/sink/filersink/filer_sink.go | 24 ++++++------- weed/replication/sink/gcssink/gcs_sink.go | 6 ++-- weed/replication/sink/s3sink/s3_sink.go | 8 ++--- weed/replication/sink/s3sink/s3_write.go | 6 ++-- weed/s3api/filer_multipart.go | 8 ++--- weed/s3api/s3api_objects_list_handlers.go | 6 ++-- weed/server/filer_grpc_server.go | 34 +++++++++---------- weed/server/filer_grpc_server_rename.go | 14 ++++---- weed/server/filer_grpc_server_sub_meta.go | 6 ++-- weed/server/filer_server.go | 24 ++++++------- weed/server/filer_server_handlers_read.go | 8 ++--- .../filer_server_handlers_write_autochunk.go | 12 +++---- .../filer_server_handlers_write_cipher.go | 6 ++-- weed/server/webdav_server.go | 20 +++++------ weed/shell/command_fs_cat.go | 4 +-- weed/shell/command_fs_du.go | 4 +-- weed/shell/command_fs_ls.go | 4 +-- weed/shell/command_volume_fsck.go | 4 +-- 76 files changed, 283 insertions(+), 283 deletions(-) rename weed/{filer2 => filer}/abstract_sql/abstract_sql_store.go (94%) rename weed/{filer2 => filer}/cassandra/README.txt (100%) rename weed/{filer2 => filer}/cassandra/cassandra_store.go (90%) rename weed/{filer2 => filer}/configuration.go (98%) rename weed/{filer2 => filer}/entry.go (99%) rename weed/{filer2 => filer}/entry_codec.go (99%) rename weed/{filer2 => filer}/etcd/etcd_store.go (91%) rename weed/{filer2 => filer}/filechunk_manifest.go (99%) rename weed/{filer2 => filer}/filechunk_manifest_test.go (99%) rename weed/{filer2 => filer}/filechunks.go (99%) rename weed/{filer2 => filer}/filechunks2_test.go (99%) rename weed/{filer2 => filer}/filechunks_test.go (99%) rename weed/{filer2 => filer}/filer.go (99%) rename weed/{filer2 => filer}/filer_buckets.go (99%) rename weed/{filer2 => filer}/filer_delete_entry.go (99%) rename weed/{filer2 => filer}/filer_deletion.go (99%) rename weed/{filer2 => filer}/filer_notify.go (99%) rename weed/{filer2 => filer}/filer_notify_append.go (99%) rename weed/{filer2 => filer}/filer_notify_test.go (98%) rename weed/{filer2 => filer}/filerstore.go (99%) rename weed/{filer2 => filer}/leveldb/leveldb_store.go (92%) rename weed/{filer2 => filer}/leveldb/leveldb_store_test.go (89%) rename weed/{filer2 => filer}/leveldb2/leveldb2_local_store.go (89%) rename weed/{filer2 => filer}/leveldb2/leveldb2_store.go (93%) rename weed/{filer2 => filer}/leveldb2/leveldb2_store_test.go (89%) rename weed/{filer2 => filer}/meta_aggregator.go (99%) rename weed/{filer2 => filer}/meta_replay.go (98%) rename weed/{filer2 => filer}/mongodb/mongodb_store.go (92%) rename weed/{filer2 => filer}/mysql/mysql_store.go (93%) rename weed/{filer2 => filer}/permission.go (95%) rename weed/{filer2 => filer}/postgres/README.txt (100%) rename weed/{filer2 => filer}/postgres/postgres_store.go (93%) rename weed/{filer2 => filer}/reader_at.go (99%) rename weed/{filer2 => filer}/reader_at_test.go (99%) rename weed/{filer2 => filer}/redis/redis_cluster_store.go (90%) rename weed/{filer2 => filer}/redis/redis_store.go (87%) rename weed/{filer2 => filer}/redis/universal_redis_store.go (91%) rename weed/{filer2 => filer}/redis2/redis_cluster_store.go (90%) rename weed/{filer2 => filer}/redis2/redis_store.go (87%) rename weed/{filer2 => filer}/redis2/universal_redis_store.go (91%) rename weed/{filer2 => filer}/stream.go (99%) rename weed/{filer2 => filer}/topics.go (84%) diff --git a/unmaintained/see_log_entry/see_log_entry.go b/unmaintained/see_log_entry/see_log_entry.go index 34965f6be..45480d4dc 100644 --- a/unmaintained/see_log_entry/see_log_entry.go +++ b/unmaintained/see_log_entry/see_log_entry.go @@ -9,13 +9,13 @@ import ( "github.com/golang/protobuf/proto" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) var ( - logdataFile = flag.String("logdata", "", "log data file saved under "+ filer2.SystemLogDir) + logdataFile = flag.String("logdata", "", "log data file saved under "+ filer.SystemLogDir) ) func main() { diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go similarity index 94% rename from weed/filer2/abstract_sql/abstract_sql_store.go rename to weed/filer/abstract_sql/abstract_sql_store.go index ce6377aac..a6de2ea39 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -4,7 +4,7 @@ import ( "context" "database/sql" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -58,7 +58,7 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB { return store.DB } -func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -92,7 +92,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer2.En } -func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -112,7 +112,7 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.En return nil } -func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer2.Entry, error) { +func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) { dir, name := fullpath.DirAndName() row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) @@ -121,7 +121,7 @@ func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.Full return nil, filer_pb.ErrNotFound } - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: fullpath, } if err := entry.DecodeAttributesAndChunks(data); err != nil { @@ -163,7 +163,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { sqlText := store.SqlListExclusive if inclusive { sqlText = store.SqlListInclusive @@ -183,7 +183,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, return nil, fmt.Errorf("scan %s: %v", fullpath, err) } - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: util.NewFullPath(string(fullpath), name), } if err = entry.DecodeAttributesAndChunks(data); err != nil { @@ -197,7 +197,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) { return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") } diff --git a/weed/filer2/cassandra/README.txt b/weed/filer/cassandra/README.txt similarity index 100% rename from weed/filer2/cassandra/README.txt rename to weed/filer/cassandra/README.txt diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer/cassandra/cassandra_store.go similarity index 90% rename from weed/filer2/cassandra/cassandra_store.go rename to weed/filer/cassandra/cassandra_store.go index 4d845c2fa..fd161b1f1 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer/cassandra/cassandra_store.go @@ -5,14 +5,14 @@ import ( "fmt" "github.com/gocql/gocql" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) func init() { - filer2.Stores = append(filer2.Stores, &CassandraStore{}) + filer.Stores = append(filer.Stores, &CassandraStore{}) } type CassandraStore struct { @@ -52,7 +52,7 @@ func (store *CassandraStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -69,12 +69,12 @@ func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer2.Entr return nil } -func (store *CassandraStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *CassandraStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { return store.InsertEntry(ctx, entry) } -func (store *CassandraStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer2.Entry, err error) { +func (store *CassandraStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { dir, name := fullpath.DirAndName() var data []byte @@ -90,7 +90,7 @@ func (store *CassandraStore) FindEntry(ctx context.Context, fullpath util.FullPa return nil, filer_pb.ErrNotFound } - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, } err = entry.DecodeAttributesAndChunks(data) @@ -125,12 +125,12 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath return nil } -func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.ErrUnsupportedListDirectoryPrefixed +func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + return nil, filer.ErrUnsupportedListDirectoryPrefixed } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer2.Entry, err error) { + limit int) (entries []*filer.Entry, err error) { cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?" if inclusive { @@ -141,7 +141,7 @@ func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath var name string iter := store.session.Query(cqlStr, string(fullpath), startFileName, limit).Iter() for iter.Scan(&name, &data) { - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: util.NewFullPath(string(fullpath), name), } if decodeErr := entry.DecodeAttributesAndChunks(data); decodeErr != nil { diff --git a/weed/filer2/configuration.go b/weed/filer/configuration.go similarity index 98% rename from weed/filer2/configuration.go rename to weed/filer/configuration.go index a174117ea..3dce67d6d 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer/configuration.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "os" diff --git a/weed/filer2/entry.go b/weed/filer/entry.go similarity index 99% rename from weed/filer2/entry.go rename to weed/filer/entry.go index fedfde40d..4a73de19a 100644 --- a/weed/filer2/entry.go +++ b/weed/filer/entry.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "os" diff --git a/weed/filer2/entry_codec.go b/weed/filer/entry_codec.go similarity index 99% rename from weed/filer2/entry_codec.go rename to weed/filer/entry_codec.go index 4d615194f..fb6448b30 100644 --- a/weed/filer2/entry_codec.go +++ b/weed/filer/entry_codec.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "bytes" diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer/etcd/etcd_store.go similarity index 91% rename from weed/filer2/etcd/etcd_store.go rename to weed/filer/etcd/etcd_store.go index 6f4c3ce5c..36db4ac01 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer/etcd/etcd_store.go @@ -8,7 +8,7 @@ import ( "go.etcd.io/etcd/clientv3" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" weed_util "github.com/chrislusf/seaweedfs/weed/util" @@ -19,7 +19,7 @@ const ( ) func init() { - filer2.Stores = append(filer2.Stores, &EtcdStore{}) + filer.Stores = append(filer.Stores, &EtcdStore{}) } type EtcdStore struct { @@ -73,7 +73,7 @@ func (store *EtcdStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { key := genKey(entry.DirAndName()) value, err := entry.EncodeAttributesAndChunks() @@ -88,11 +88,11 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (e return nil } -func (store *EtcdStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *EtcdStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { return store.InsertEntry(ctx, entry) } -func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer2.Entry, err error) { +func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { key := genKey(fullpath.DirAndName()) resp, err := store.client.Get(ctx, string(key)) @@ -104,7 +104,7 @@ func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPa return nil, filer_pb.ErrNotFound } - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, } err = entry.DecodeAttributesAndChunks(resp.Kvs[0].Value) @@ -135,11 +135,11 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.ErrUnsupportedListDirectoryPrefixed +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + return nil, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { +func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, "") resp, err := store.client.Get(ctx, string(directoryPrefix), @@ -160,7 +160,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_ if limit < 0 { break } - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(fullpath), fileName), } if decodeErr := entry.DecodeAttributesAndChunks(kv.Value); decodeErr != nil { diff --git a/weed/filer2/filechunk_manifest.go b/weed/filer/filechunk_manifest.go similarity index 99% rename from weed/filer2/filechunk_manifest.go rename to weed/filer/filechunk_manifest.go index ba4625bab..e84cf21e5 100644 --- a/weed/filer2/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "bytes" diff --git a/weed/filer2/filechunk_manifest_test.go b/weed/filer/filechunk_manifest_test.go similarity index 99% rename from weed/filer2/filechunk_manifest_test.go rename to weed/filer/filechunk_manifest_test.go index 2b0862d07..ce12c5da6 100644 --- a/weed/filer2/filechunk_manifest_test.go +++ b/weed/filer/filechunk_manifest_test.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "bytes" diff --git a/weed/filer2/filechunks.go b/weed/filer/filechunks.go similarity index 99% rename from weed/filer2/filechunks.go rename to weed/filer/filechunks.go index 53c679d6b..c45963193 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer/filechunks.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "fmt" diff --git a/weed/filer2/filechunks2_test.go b/weed/filer/filechunks2_test.go similarity index 99% rename from weed/filer2/filechunks2_test.go rename to weed/filer/filechunks2_test.go index d896da3cc..9f9566d9b 100644 --- a/weed/filer2/filechunks2_test.go +++ b/weed/filer/filechunks2_test.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "sort" diff --git a/weed/filer2/filechunks_test.go b/weed/filer/filechunks_test.go similarity index 99% rename from weed/filer2/filechunks_test.go rename to weed/filer/filechunks_test.go index 31b74a22a..699e7e298 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer/filechunks_test.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "fmt" diff --git a/weed/filer2/filer.go b/weed/filer/filer.go similarity index 99% rename from weed/filer2/filer.go rename to weed/filer/filer.go index 78e27ed08..71da1a080 100644 --- a/weed/filer2/filer.go +++ b/weed/filer/filer.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/filer_buckets.go b/weed/filer/filer_buckets.go similarity index 99% rename from weed/filer2/filer_buckets.go rename to weed/filer/filer_buckets.go index 6b7c2c31a..4d4f4abc3 100644 --- a/weed/filer2/filer_buckets.go +++ b/weed/filer/filer_buckets.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/filer_delete_entry.go b/weed/filer/filer_delete_entry.go similarity index 99% rename from weed/filer2/filer_delete_entry.go rename to weed/filer/filer_delete_entry.go index 4d2a1ef43..e2198bd21 100644 --- a/weed/filer2/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/filer_deletion.go b/weed/filer/filer_deletion.go similarity index 99% rename from weed/filer2/filer_deletion.go rename to weed/filer/filer_deletion.go index dbee4a61d..126d162ec 100644 --- a/weed/filer2/filer_deletion.go +++ b/weed/filer/filer_deletion.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "strings" diff --git a/weed/filer2/filer_notify.go b/weed/filer/filer_notify.go similarity index 99% rename from weed/filer2/filer_notify.go rename to weed/filer/filer_notify.go index a95fcf0b4..e00117382 100644 --- a/weed/filer2/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/filer_notify_append.go b/weed/filer/filer_notify_append.go similarity index 99% rename from weed/filer2/filer_notify_append.go rename to weed/filer/filer_notify_append.go index 31cdb3c1c..b1836b046 100644 --- a/weed/filer2/filer_notify_append.go +++ b/weed/filer/filer_notify_append.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/filer_notify_test.go b/weed/filer/filer_notify_test.go similarity index 98% rename from weed/filer2/filer_notify_test.go rename to weed/filer/filer_notify_test.go index 29170bfdf..6a2be8f18 100644 --- a/weed/filer2/filer_notify_test.go +++ b/weed/filer/filer_notify_test.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "testing" diff --git a/weed/filer2/filerstore.go b/weed/filer/filerstore.go similarity index 99% rename from weed/filer2/filerstore.go rename to weed/filer/filerstore.go index dd5abe421..48f7c99e4 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer/filerstore.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer/leveldb/leveldb_store.go similarity index 92% rename from weed/filer2/leveldb/leveldb_store.go rename to weed/filer/leveldb/leveldb_store.go index 1c08d2831..eccb760a2 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer/leveldb/leveldb_store.go @@ -9,7 +9,7 @@ import ( "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" weed_util "github.com/chrislusf/seaweedfs/weed/util" @@ -20,7 +20,7 @@ const ( ) func init() { - filer2.Stores = append(filer2.Stores, &LevelDBStore{}) + filer.Stores = append(filer.Stores, &LevelDBStore{}) } type LevelDBStore struct { @@ -70,7 +70,7 @@ func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { key := genKey(entry.DirAndName()) value, err := entry.EncodeAttributesAndChunks() @@ -89,12 +89,12 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer2.Entry) return nil } -func (store *LevelDBStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *LevelDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { return store.InsertEntry(ctx, entry) } -func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer2.Entry, err error) { +func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { key := genKey(fullpath.DirAndName()) data, err := store.db.Get(key, nil) @@ -106,7 +106,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) } - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, } err = entry.DecodeAttributesAndChunks(data) @@ -158,12 +158,12 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.ErrUnsupportedListDirectoryPrefixed +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + return nil, filer.ErrUnsupportedListDirectoryPrefixed } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer2.Entry, err error) { + limit int) (entries []*filer.Entry, err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, "") @@ -184,7 +184,7 @@ func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath we if limit < 0 { break } - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(fullpath), fileName), } if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil { diff --git a/weed/filer2/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go similarity index 89% rename from weed/filer2/leveldb/leveldb_store_test.go rename to weed/filer/leveldb/leveldb_store_test.go index 86e8aa952..df196b02e 100644 --- a/weed/filer2/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -6,12 +6,12 @@ import ( "os" "testing" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" ) func TestCreateAndFind(t *testing.T) { - filer := filer2.NewFiler(nil, nil, "", 0, "", "", nil) + filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") defer os.RemoveAll(dir) store := &LevelDBStore{} @@ -22,9 +22,9 @@ func TestCreateAndFind(t *testing.T) { ctx := context.Background() - entry1 := &filer2.Entry{ + entry1 := &filer.Entry{ FullPath: fullpath, - Attr: filer2.Attr{ + Attr: filer.Attr{ Mode: 0440, Uid: 1234, Gid: 5678, @@ -65,7 +65,7 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - filer := filer2.NewFiler(nil, nil, "", 0, "", "", nil) + filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") defer os.RemoveAll(dir) store := &LevelDBStore{} diff --git a/weed/filer2/leveldb2/leveldb2_local_store.go b/weed/filer/leveldb2/leveldb2_local_store.go similarity index 89% rename from weed/filer2/leveldb2/leveldb2_local_store.go rename to weed/filer/leveldb2/leveldb2_local_store.go index 3625abf9e..faae25c45 100644 --- a/weed/filer2/leveldb2/leveldb2_local_store.go +++ b/weed/filer/leveldb2/leveldb2_local_store.go @@ -3,12 +3,12 @@ package leveldb import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" ) var ( - _ = filer2.FilerLocalStore(&LevelDB2Store{}) + _ = filer.FilerLocalStore(&LevelDB2Store{}) ) func (store *LevelDB2Store) UpdateOffset(filer string, lastTsNs int64) error { diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go similarity index 93% rename from weed/filer2/leveldb2/leveldb2_store.go rename to weed/filer/leveldb2/leveldb2_store.go index ca9d6f04d..7a2bdac2e 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -12,14 +12,14 @@ import ( "io" "os" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" weed_util "github.com/chrislusf/seaweedfs/weed/util" ) func init() { - filer2.Stores = append(filer2.Stores, &LevelDB2Store{}) + filer.Stores = append(filer.Stores, &LevelDB2Store{}) } type LevelDB2Store struct { @@ -76,7 +76,7 @@ func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error { return nil } -func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.DirAndName() key, partitionId := genKey(dir, name, store.dbCount) @@ -96,12 +96,12 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer2.Entry return nil } -func (store *LevelDB2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *LevelDB2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { return store.InsertEntry(ctx, entry) } -func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer2.Entry, err error) { +func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { dir, name := fullpath.DirAndName() key, partitionId := genKey(dir, name, store.dbCount) @@ -114,7 +114,7 @@ func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath weed_util.Fu return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) } - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, } err = entry.DecodeAttributesAndChunks(data) @@ -167,12 +167,12 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.ErrUnsupportedListDirectoryPrefixed +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + return nil, filer.ErrUnsupportedListDirectoryPrefixed } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer2.Entry, err error) { + limit int) (entries []*filer.Entry, err error) { directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, "", store.dbCount) lastFileStart, _ := genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount) @@ -194,7 +194,7 @@ func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath w if limit < 0 { break } - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: weed_util.NewFullPath(string(fullpath), fileName), } diff --git a/weed/filer2/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go similarity index 89% rename from weed/filer2/leveldb2/leveldb2_store_test.go rename to weed/filer/leveldb2/leveldb2_store_test.go index 62d2475fe..191de0040 100644 --- a/weed/filer2/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -6,12 +6,12 @@ import ( "os" "testing" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" ) func TestCreateAndFind(t *testing.T) { - filer := filer2.NewFiler(nil, nil, "", 0, "", "", nil) + filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") defer os.RemoveAll(dir) store := &LevelDB2Store{} @@ -22,9 +22,9 @@ func TestCreateAndFind(t *testing.T) { ctx := context.Background() - entry1 := &filer2.Entry{ + entry1 := &filer.Entry{ FullPath: fullpath, - Attr: filer2.Attr{ + Attr: filer.Attr{ Mode: 0440, Uid: 1234, Gid: 5678, @@ -65,7 +65,7 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - filer := filer2.NewFiler(nil, nil, "", 0, "", "", nil) + filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") defer os.RemoveAll(dir) store := &LevelDB2Store{} diff --git a/weed/filer2/meta_aggregator.go b/weed/filer/meta_aggregator.go similarity index 99% rename from weed/filer2/meta_aggregator.go rename to weed/filer/meta_aggregator.go index f2792bd26..506f03e4c 100644 --- a/weed/filer2/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/meta_replay.go b/weed/filer/meta_replay.go similarity index 98% rename from weed/filer2/meta_replay.go rename to weed/filer/meta_replay.go index d9cdaa76a..feb76278b 100644 --- a/weed/filer2/meta_replay.go +++ b/weed/filer/meta_replay.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go similarity index 92% rename from weed/filer2/mongodb/mongodb_store.go rename to weed/filer/mongodb/mongodb_store.go index 661aa4ea0..57d9a031d 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -3,7 +3,7 @@ package mongodb import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -15,7 +15,7 @@ import ( ) func init() { - filer2.Stores = append(filer2.Stores, &MongodbStore{}) + filer.Stores = append(filer.Stores, &MongodbStore{}) } type MongodbStore struct { @@ -93,7 +93,7 @@ func (store *MongodbStore) RollbackTransaction(ctx context.Context) error { return nil } -func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() @@ -112,11 +112,11 @@ func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) return nil } -func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { return store.InsertEntry(ctx, entry) } -func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer2.Entry, err error) { +func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { dir, name := fullpath.DirAndName() var data Model @@ -131,7 +131,7 @@ func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath return nil, filer_pb.ErrNotFound } - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, } @@ -167,11 +167,11 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut return nil } -func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.ErrUnsupportedListDirectoryPrefixed +func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + return nil, filer.ErrUnsupportedListDirectoryPrefixed } -func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { +func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer.Entry, err error) { var where = bson.M{"directory": string(fullpath), "name": bson.M{"$gt": startFileName}} if inclusive { @@ -189,7 +189,7 @@ func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath ut return nil, err } - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: util.NewFullPath(string(fullpath), data.Name), } if decodeErr := entry.DecodeAttributesAndChunks(data.Meta); decodeErr != nil { diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go similarity index 93% rename from weed/filer2/mysql/mysql_store.go rename to weed/filer/mysql/mysql_store.go index 9f748445e..708a67cc3 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -4,8 +4,8 @@ import ( "database/sql" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/util" _ "github.com/go-sql-driver/mysql" ) @@ -15,7 +15,7 @@ const ( ) func init() { - filer2.Stores = append(filer2.Stores, &MysqlStore{}) + filer.Stores = append(filer.Stores, &MysqlStore{}) } type MysqlStore struct { diff --git a/weed/filer2/permission.go b/weed/filer/permission.go similarity index 95% rename from weed/filer2/permission.go rename to weed/filer/permission.go index 8a9508fbc..0d8b8292b 100644 --- a/weed/filer2/permission.go +++ b/weed/filer/permission.go @@ -1,4 +1,4 @@ -package filer2 +package filer func hasWritePermission(dir *Entry, entry *Entry) bool { diff --git a/weed/filer2/postgres/README.txt b/weed/filer/postgres/README.txt similarity index 100% rename from weed/filer2/postgres/README.txt rename to weed/filer/postgres/README.txt diff --git a/weed/filer2/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go similarity index 93% rename from weed/filer2/postgres/postgres_store.go rename to weed/filer/postgres/postgres_store.go index 87eb6aca2..4544c8416 100644 --- a/weed/filer2/postgres/postgres_store.go +++ b/weed/filer/postgres/postgres_store.go @@ -4,8 +4,8 @@ import ( "database/sql" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/util" _ "github.com/lib/pq" ) @@ -15,7 +15,7 @@ const ( ) func init() { - filer2.Stores = append(filer2.Stores, &PostgresStore{}) + filer.Stores = append(filer.Stores, &PostgresStore{}) } type PostgresStore struct { diff --git a/weed/filer2/reader_at.go b/weed/filer/reader_at.go similarity index 99% rename from weed/filer2/reader_at.go rename to weed/filer/reader_at.go index 0cea83ff9..9f338782e 100644 --- a/weed/filer2/reader_at.go +++ b/weed/filer/reader_at.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "context" diff --git a/weed/filer2/reader_at_test.go b/weed/filer/reader_at_test.go similarity index 99% rename from weed/filer2/reader_at_test.go rename to weed/filer/reader_at_test.go index 7bfc9a972..d4a34cbfe 100644 --- a/weed/filer2/reader_at_test.go +++ b/weed/filer/reader_at_test.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "fmt" diff --git a/weed/filer2/redis/redis_cluster_store.go b/weed/filer/redis/redis_cluster_store.go similarity index 90% rename from weed/filer2/redis/redis_cluster_store.go rename to weed/filer/redis/redis_cluster_store.go index eaaecb740..8af94ee55 100644 --- a/weed/filer2/redis/redis_cluster_store.go +++ b/weed/filer/redis/redis_cluster_store.go @@ -1,13 +1,13 @@ package redis import ( - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" "github.com/go-redis/redis" ) func init() { - filer2.Stores = append(filer2.Stores, &RedisClusterStore{}) + filer.Stores = append(filer.Stores, &RedisClusterStore{}) } type RedisClusterStore struct { diff --git a/weed/filer2/redis/redis_store.go b/weed/filer/redis/redis_store.go similarity index 87% rename from weed/filer2/redis/redis_store.go rename to weed/filer/redis/redis_store.go index 9debdb070..e152457ed 100644 --- a/weed/filer2/redis/redis_store.go +++ b/weed/filer/redis/redis_store.go @@ -1,13 +1,13 @@ package redis import ( - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" "github.com/go-redis/redis" ) func init() { - filer2.Stores = append(filer2.Stores, &RedisStore{}) + filer.Stores = append(filer.Stores, &RedisStore{}) } type RedisStore struct { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go similarity index 91% rename from weed/filer2/redis/universal_redis_store.go rename to weed/filer/redis/universal_redis_store.go index fc2abef6c..cc8819019 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -9,7 +9,7 @@ import ( "github.com/go-redis/redis" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -33,7 +33,7 @@ func (store *UniversalRedisStore) RollbackTransaction(ctx context.Context) error return nil } -func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { value, err := entry.EncodeAttributesAndChunks() if err != nil { @@ -57,12 +57,12 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer2 return nil } -func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { return store.InsertEntry(ctx, entry) } -func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer2.Entry, err error) { +func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { data, err := store.Client.Get(string(fullpath)).Result() if err == redis.Nil { @@ -73,7 +73,7 @@ func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.F return nil, fmt.Errorf("get %s : %v", fullpath, err) } - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, } err = entry.DecodeAttributesAndChunks([]byte(data)) @@ -121,12 +121,12 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full return nil } -func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.ErrUnsupportedListDirectoryPrefixed +func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + return nil, filer.ErrUnsupportedListDirectoryPrefixed } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer2.Entry, err error) { + limit int) (entries []*filer.Entry, err error) { dirListKey := genDirectoryListKey(string(fullpath)) members, err := store.Client.SMembers(dirListKey).Result() diff --git a/weed/filer2/redis2/redis_cluster_store.go b/weed/filer/redis2/redis_cluster_store.go similarity index 90% rename from weed/filer2/redis2/redis_cluster_store.go rename to weed/filer/redis2/redis_cluster_store.go index b252eabab..d155dbe88 100644 --- a/weed/filer2/redis2/redis_cluster_store.go +++ b/weed/filer/redis2/redis_cluster_store.go @@ -1,13 +1,13 @@ package redis2 import ( - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" "github.com/go-redis/redis" ) func init() { - filer2.Stores = append(filer2.Stores, &RedisCluster2Store{}) + filer.Stores = append(filer.Stores, &RedisCluster2Store{}) } type RedisCluster2Store struct { diff --git a/weed/filer2/redis2/redis_store.go b/weed/filer/redis2/redis_store.go similarity index 87% rename from weed/filer2/redis2/redis_store.go rename to weed/filer/redis2/redis_store.go index 1e2a20043..ed04c817b 100644 --- a/weed/filer2/redis2/redis_store.go +++ b/weed/filer/redis2/redis_store.go @@ -1,13 +1,13 @@ package redis2 import ( - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" "github.com/go-redis/redis" ) func init() { - filer2.Stores = append(filer2.Stores, &Redis2Store{}) + filer.Stores = append(filer.Stores, &Redis2Store{}) } type Redis2Store struct { diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go similarity index 91% rename from weed/filer2/redis2/universal_redis_store.go rename to weed/filer/redis2/universal_redis_store.go index c639635ef..9e06ff68f 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -7,7 +7,7 @@ import ( "github.com/go-redis/redis" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -31,7 +31,7 @@ func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) erro return nil } -func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { value, err := entry.EncodeAttributesAndChunks() if err != nil { @@ -52,12 +52,12 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer return nil } -func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { +func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { return store.InsertEntry(ctx, entry) } -func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer2.Entry, err error) { +func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { data, err := store.Client.Get(string(fullpath)).Result() if err == redis.Nil { @@ -68,7 +68,7 @@ func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util. return nil, fmt.Errorf("get %s : %v", fullpath, err) } - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, } err = entry.DecodeAttributesAndChunks([]byte(data)) @@ -116,12 +116,12 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } -func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.ErrUnsupportedListDirectoryPrefixed +func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) { + return nil, filer.ErrUnsupportedListDirectoryPrefixed } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, - limit int) (entries []*filer2.Entry, err error) { + limit int) (entries []*filer.Entry, err error) { dirListKey := genDirectoryListKey(string(fullpath)) start := int64(0) diff --git a/weed/filer2/stream.go b/weed/filer/stream.go similarity index 99% rename from weed/filer2/stream.go rename to weed/filer/stream.go index fee9d45da..416359ebf 100644 --- a/weed/filer2/stream.go +++ b/weed/filer/stream.go @@ -1,4 +1,4 @@ -package filer2 +package filer import ( "bytes" diff --git a/weed/filer2/topics.go b/weed/filer/topics.go similarity index 84% rename from weed/filer2/topics.go rename to weed/filer/topics.go index 9c6e5c88d..3a2fde8c4 100644 --- a/weed/filer2/topics.go +++ b/weed/filer/topics.go @@ -1,4 +1,4 @@ -package filer2 +package filer const ( TopicsDir = "/topics" diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index f85b90a5d..59c4b7965 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -11,7 +11,7 @@ import ( "github.com/seaweedfs/fuse" "github.com/seaweedfs/fuse/fs" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filesys/meta_cache" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -156,7 +156,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, return fuse.EIO } - dir.wfs.metaCache.InsertEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry)) + dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil }); err != nil { @@ -205,7 +205,7 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err return err } - dir.wfs.metaCache.InsertEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry)) + dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil }) @@ -471,7 +471,7 @@ func (dir *Dir) saveEntry() error { return fuse.EIO } - dir.wfs.metaCache.UpdateEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry)) + dir.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil }) diff --git a/weed/filesys/dir_link.go b/weed/filesys/dir_link.go index d813dd96a..71aa193f1 100644 --- a/weed/filesys/dir_link.go +++ b/weed/filesys/dir_link.go @@ -6,7 +6,7 @@ import ( "syscall" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/fuse" @@ -43,7 +43,7 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, return fuse.EIO } - dir.wfs.metaCache.InsertEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry)) + dir.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil }) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index d2117bfbb..abc2935c5 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -10,7 +10,7 @@ import ( "github.com/seaweedfs/fuse" "github.com/seaweedfs/fuse/fs" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -33,7 +33,7 @@ type File struct { dir *Dir wfs *WFS entry *filer_pb.Entry - entryViewCache []filer2.VisibleInterval + entryViewCache []filer.VisibleInterval isOpen int reader io.ReaderAt dirtyMetadata bool @@ -56,7 +56,7 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { attr.Inode = file.fullpath().AsInode() attr.Valid = time.Second attr.Mode = os.FileMode(file.entry.Attributes.FileMode) - attr.Size = filer2.FileSize(file.entry) + attr.Size = filer.FileSize(file.entry) if file.isOpen > 0 { attr.Size = file.entry.Attributes.FileSize glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size) @@ -118,7 +118,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f if req.Valid.Size() { glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(file.entry.Chunks)) - if req.Size < filer2.FileSize(file.entry) { + if req.Size < filer.FileSize(file.entry) { // fmt.Printf("truncate %v \n", fullPath) var chunks []*filer_pb.FileChunk var truncatedChunks []*filer_pb.FileChunk @@ -273,7 +273,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) { }) for _, chunk := range chunks { - file.entryViewCache = filer2.MergeIntoVisibles(file.entryViewCache, chunk) + file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk) } file.reader = nil @@ -285,7 +285,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) { func (file *File) setEntry(entry *filer_pb.Entry) { file.entry = entry - file.entryViewCache, _ = filer2.NonOverlappingVisibleIntervals(filer2.LookupFn(file.wfs), file.entry.Chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), file.entry.Chunks) file.reader = nil } @@ -305,7 +305,7 @@ func (file *File) saveEntry() error { return fuse.EIO } - file.wfs.metaCache.UpdateEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry)) + file.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil }) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index a1f18df6f..195d8ae8d 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -13,7 +13,7 @@ import ( "github.com/seaweedfs/fuse" "github.com/seaweedfs/fuse/fs" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -41,7 +41,7 @@ func newFileHandle(file *File, uid, gid uint32) *FileHandle { Gid: gid, } if fh.f.entry != nil { - fh.f.entry.Attributes.FileSize = filer2.FileSize(fh.f.entry) + fh.f.entry.Attributes.FileSize = filer.FileSize(fh.f.entry) } return fh @@ -99,7 +99,7 @@ func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxSto func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { - fileSize := int64(filer2.FileSize(fh.f.entry)) + fileSize := int64(filer.FileSize(fh.f.entry)) if fileSize == 0 { glog.V(1).Infof("empty fh %v", fh.f.fullpath()) @@ -108,7 +108,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { var chunkResolveErr error if fh.f.entryViewCache == nil { - fh.f.entryViewCache, chunkResolveErr = filer2.NonOverlappingVisibleIntervals(filer2.LookupFn(fh.f.wfs), fh.f.entry.Chunks) + fh.f.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(filer.LookupFn(fh.f.wfs), fh.f.entry.Chunks) if chunkResolveErr != nil { return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr) } @@ -116,8 +116,8 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { } if fh.f.reader == nil { - chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64) - fh.f.reader = filer2.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize) + chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64) + fh.f.reader = filer.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize) } totalRead, err := fh.f.reader.ReadAt(buff, offset) @@ -254,10 +254,10 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) } - manifestChunks, nonManifestChunks := filer2.SeparateManifestChunks(fh.f.entry.Chunks) + manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(fh.f.entry.Chunks) - chunks, _ := filer2.CompactFileChunks(filer2.LookupFn(fh.f.wfs), nonManifestChunks) - chunks, manifestErr := filer2.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.dir.FullPath()), chunks) + chunks, _ := filer.CompactFileChunks(filer.LookupFn(fh.f.wfs), nonManifestChunks) + chunks, manifestErr := filer.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.dir.FullPath()), chunks) if manifestErr != nil { // not good, but should be ok glog.V(0).Infof("MaybeManifestize: %v", manifestErr) @@ -270,7 +270,7 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error { return fmt.Errorf("fh flush create %s: %v", fh.f.fullpath(), err) } - fh.f.wfs.metaCache.InsertEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry)) + fh.f.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) return nil }) diff --git a/weed/filesys/meta_cache/meta_cache.go b/weed/filesys/meta_cache/meta_cache.go index c0bb75f4a..f714fde09 100644 --- a/weed/filesys/meta_cache/meta_cache.go +++ b/weed/filesys/meta_cache/meta_cache.go @@ -5,8 +5,8 @@ import ( "os" "sync" - "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/leveldb" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -17,7 +17,7 @@ import ( // e.g. fill fileId field for chunks type MetaCache struct { - actualStore filer2.FilerStore + actualStore filer.FilerStore sync.RWMutex visitedBoundary *bounded_tree.BoundedTree } @@ -29,7 +29,7 @@ func NewMetaCache(dbFolder string) *MetaCache { } } -func openMetaStore(dbFolder string) filer2.FilerStore { +func openMetaStore(dbFolder string) filer.FilerStore { os.RemoveAll(dbFolder) os.MkdirAll(dbFolder, 0755) @@ -47,18 +47,18 @@ func openMetaStore(dbFolder string) filer2.FilerStore { } -func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer2.Entry) error { +func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error { mc.Lock() defer mc.Unlock() return mc.doInsertEntry(ctx, entry) } -func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer2.Entry) error { +func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) error { filer_pb.BeforeEntrySerialization(entry.Chunks) return mc.actualStore.InsertEntry(ctx, entry) } -func (mc *MetaCache) AtomicUpdateEntry(ctx context.Context, oldPath util.FullPath, newEntry *filer2.Entry) error { +func (mc *MetaCache) AtomicUpdateEntry(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error { mc.Lock() defer mc.Unlock() @@ -89,14 +89,14 @@ func (mc *MetaCache) AtomicUpdateEntry(ctx context.Context, oldPath util.FullPat return nil } -func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer2.Entry) error { +func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error { mc.Lock() defer mc.Unlock() filer_pb.BeforeEntrySerialization(entry.Chunks) return mc.actualStore.UpdateEntry(ctx, entry) } -func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer2.Entry, err error) { +func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, err error) { mc.RLock() defer mc.RUnlock() entry, err = mc.actualStore.FindEntry(ctx, fp) @@ -113,7 +113,7 @@ func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err err return mc.actualStore.DeleteEntry(ctx, fp) } -func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer2.Entry, error) { +func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*filer.Entry, error) { mc.RLock() defer mc.RUnlock() diff --git a/weed/filesys/meta_cache/meta_cache_init.go b/weed/filesys/meta_cache/meta_cache_init.go index 05983ec7d..455a8772c 100644 --- a/weed/filesys/meta_cache/meta_cache_init.go +++ b/weed/filesys/meta_cache/meta_cache_init.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -17,7 +17,7 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full glog.V(4).Infof("ReadDirAllEntries %s ...", path) err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error { - entry := filer2.FromPbEntry(string(dirPath), pbEntry) + entry := filer.FromPbEntry(string(dirPath), pbEntry) if err := mc.doInsertEntry(context.Background(), entry); err != nil { glog.V(0).Infof("read %s: %v", entry.FullPath, err) return err diff --git a/weed/filesys/meta_cache/meta_cache_subscribe.go b/weed/filesys/meta_cache/meta_cache_subscribe.go index 3c0a9c2ac..bd98666ed 100644 --- a/weed/filesys/meta_cache/meta_cache_subscribe.go +++ b/weed/filesys/meta_cache/meta_cache_subscribe.go @@ -6,7 +6,7 @@ import ( "io" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -24,7 +24,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil } var oldPath util.FullPath - var newEntry *filer2.Entry + var newEntry *filer.Entry if message.OldEntry != nil { oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name) glog.V(4).Infof("deleting %v", oldPath) @@ -37,7 +37,7 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil } key := util.NewFullPath(dir, message.NewEntry.Name) glog.V(4).Infof("creating %v", key) - newEntry = filer2.FromPbEntry(dir, message.NewEntry) + newEntry = filer.FromPbEntry(dir, message.NewEntry) } return mc.AtomicUpdateEntry(context.Background(), oldPath, newEntry) } diff --git a/weed/filesys/wfs_deletion.go b/weed/filesys/wfs_deletion.go index 87a4e907f..9791c8630 100644 --- a/weed/filesys/wfs_deletion.go +++ b/weed/filesys/wfs_deletion.go @@ -5,7 +5,7 @@ import ( "google.golang.org/grpc" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -22,7 +22,7 @@ func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) { fileIds = append(fileIds, chunk.GetFileIdString()) continue } - dataChunks, manifestResolveErr := filer2.ResolveOneChunkManifest(filer2.LookupFn(wfs), chunk) + dataChunks, manifestResolveErr := filer.ResolveOneChunkManifest(filer.LookupFn(wfs), chunk) if manifestResolveErr != nil { glog.V(0).Infof("failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr) } @@ -42,7 +42,7 @@ func (wfs *WFS) deleteFileIds(grpcDialOption grpc.DialOption, client filer_pb.Se var vids []string for _, fileId := range fileIds { - vids = append(vids, filer2.VolumeId(fileId)) + vids = append(vids, filer.VolumeId(fileId)) } lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) { diff --git a/weed/filesys/wfs_write.go b/weed/filesys/wfs_write.go index 786d0b42a..fec33e4ab 100644 --- a/weed/filesys/wfs_write.go +++ b/weed/filesys/wfs_write.go @@ -5,14 +5,14 @@ import ( "fmt" "io" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/security" ) -func (wfs *WFS) saveDataAsChunk(dir string) filer2.SaveDataAsChunkFunctionType { +func (wfs *WFS) saveDataAsChunk(dir string) filer.SaveDataAsChunkFunctionType { return func(reader io.Reader, filename string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) { var fileId, host string diff --git a/weed/messaging/broker/broker_grpc_server.go b/weed/messaging/broker/broker_grpc_server.go index 1950326ec..8e207b1cc 100644 --- a/weed/messaging/broker/broker_grpc_server.go +++ b/weed/messaging/broker/broker_grpc_server.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" ) @@ -29,9 +29,9 @@ func (broker *MessageBroker) GetTopicConfiguration(c context.Context, request *m } func genTopicDir(namespace, topic string) string { - return fmt.Sprintf("%s/%s/%s", filer2.TopicsDir, namespace, topic) + return fmt.Sprintf("%s/%s/%s", filer.TopicsDir, namespace, topic) } func genTopicDirEntry(namespace, topic string) (dir, entry string) { - return fmt.Sprintf("%s/%s", filer2.TopicsDir, namespace), topic + return fmt.Sprintf("%s/%s", filer.TopicsDir, namespace), topic } diff --git a/weed/messaging/broker/broker_grpc_server_publish.go b/weed/messaging/broker/broker_grpc_server_publish.go index 154bf8a44..6e6b723d1 100644 --- a/weed/messaging/broker/broker_grpc_server_publish.go +++ b/weed/messaging/broker/broker_grpc_server_publish.go @@ -7,7 +7,7 @@ import ( "github.com/golang/protobuf/proto" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" @@ -49,7 +49,7 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis Partition: in.Init.Partition, } - tpDir := fmt.Sprintf("%s/%s/%s", filer2.TopicsDir, tp.Namespace, tp.Topic) + tpDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, tp.Namespace, tp.Topic) md5File := fmt.Sprintf("p%02d.md5", tp.Partition) // println("chan data stored under", tpDir, "as", md5File) diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 8cc5a928c..4a89937c1 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -8,7 +8,7 @@ import ( "github.com/golang/protobuf/proto" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" @@ -147,9 +147,9 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim return nil } // println("partition", tp.Partition, "processing", dayDir, "/", hourMinuteEntry.Name) - chunkedFileReader := filer2.NewChunkStreamReader(broker, hourMinuteEntry.Chunks) + chunkedFileReader := filer.NewChunkStreamReader(broker, hourMinuteEntry.Chunks) defer chunkedFileReader.Close() - if _, err := filer2.ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil { + if _, err := filer.ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil { chunkedFileReader.Close() if err == io.EOF { return err diff --git a/weed/messaging/broker/topic_manager.go b/weed/messaging/broker/topic_manager.go index 93815f8f4..edddca813 100644 --- a/weed/messaging/broker/topic_manager.go +++ b/weed/messaging/broker/topic_manager.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" "github.com/chrislusf/seaweedfs/weed/util/log_buffer" @@ -59,7 +59,7 @@ func (tm *TopicManager) buildLogBuffer(tl *TopicControl, tp TopicPartition, topi startTime, stopTime = startTime.UTC(), stopTime.UTC() targetFile := fmt.Sprintf( "%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d", - filer2.TopicsDir, tp.Namespace, tp.Topic, + filer.TopicsDir, tp.Namespace, tp.Topic, startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), tp.Partition, ) diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index 6419509be..64c3d46ea 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/replication/sink" @@ -95,8 +95,8 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error { return nil } - totalSize := filer2.FileSize(entry) - chunkViews := filer2.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) + totalSize := filer.FileSize(entry) + chunkViews := filer.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) // Create a URL that references a to-be-created blob in your // Azure Storage account's container. diff --git a/weed/replication/sink/b2sink/b2_sink.go b/weed/replication/sink/b2sink/b2_sink.go index 041cee952..d0b3e7a34 100644 --- a/weed/replication/sink/b2sink/b2_sink.go +++ b/weed/replication/sink/b2sink/b2_sink.go @@ -4,7 +4,7 @@ import ( "context" "strings" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/replication/sink" "github.com/chrislusf/seaweedfs/weed/replication/source" @@ -84,8 +84,8 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error { return nil } - totalSize := filer2.FileSize(entry) - chunkViews := filer2.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) + totalSize := filer.FileSize(entry) + chunkViews := filer.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) bucket, err := g.client.Bucket(context.Background(), g.bucket) if err != nil { diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go index b90a642c9..7ba1670e0 100644 --- a/weed/replication/sink/filersink/filer_sink.go +++ b/weed/replication/sink/filersink/filer_sink.go @@ -8,7 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/replication/sink" @@ -92,7 +92,7 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry) error { } glog.V(1).Infof("lookup: %v", lookupRequest) if resp, err := filer_pb.LookupEntry(client, lookupRequest); err == nil { - if filer2.ETag(resp.Entry) == filer2.ETag(entry) { + if filer.ETag(resp.Entry) == filer.ETag(entry) { glog.V(0).Infof("already replicated %s", key) return nil } @@ -164,13 +164,13 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent // skip if already changed // this usually happens when the messages are not ordered glog.V(0).Infof("late updates %s", key) - } else if filer2.ETag(newEntry) == filer2.ETag(existingEntry) { + } else if filer.ETag(newEntry) == filer.ETag(existingEntry) { // skip if no change // this usually happens when retrying the replication glog.V(0).Infof("already replicated %s", key) } else { // find out what changed - deletedChunks, newChunks, err := compareChunks(filer2.LookupFn(fs), oldEntry, newEntry) + deletedChunks, newChunks, err := compareChunks(filer.LookupFn(fs), oldEntry, newEntry) if err != nil { return true, fmt.Errorf("replicte %s compare chunks error: %v", key, err) } @@ -178,7 +178,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent // delete the chunks that are deleted from the source if deleteIncludeChunks { // remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks - existingEntry.Chunks = filer2.DoMinusChunks(existingEntry.Chunks, deletedChunks) + existingEntry.Chunks = filer.DoMinusChunks(existingEntry.Chunks, deletedChunks) } // replicate the chunks that are new in the source @@ -207,21 +207,21 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent }) } -func compareChunks(lookupFileIdFn filer2.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) { - aData, aMeta, aErr := filer2.ResolveChunkManifest(lookupFileIdFn, oldEntry.Chunks) +func compareChunks(lookupFileIdFn filer.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) { + aData, aMeta, aErr := filer.ResolveChunkManifest(lookupFileIdFn, oldEntry.Chunks) if aErr != nil { return nil, nil, aErr } - bData, bMeta, bErr := filer2.ResolveChunkManifest(lookupFileIdFn, newEntry.Chunks) + bData, bMeta, bErr := filer.ResolveChunkManifest(lookupFileIdFn, newEntry.Chunks) if bErr != nil { return nil, nil, bErr } - deletedChunks = append(deletedChunks, filer2.DoMinusChunks(aData, bData)...) - deletedChunks = append(deletedChunks, filer2.DoMinusChunks(aMeta, bMeta)...) + deletedChunks = append(deletedChunks, filer.DoMinusChunks(aData, bData)...) + deletedChunks = append(deletedChunks, filer.DoMinusChunks(aMeta, bMeta)...) - newChunks = append(newChunks, filer2.DoMinusChunks(bData, aData)...) - newChunks = append(newChunks, filer2.DoMinusChunks(bMeta, aMeta)...) + newChunks = append(newChunks, filer.DoMinusChunks(bData, aData)...) + newChunks = append(newChunks, filer.DoMinusChunks(bMeta, aMeta)...) return } diff --git a/weed/replication/sink/gcssink/gcs_sink.go b/weed/replication/sink/gcssink/gcs_sink.go index 82f4d72cf..2e09a87f9 100644 --- a/weed/replication/sink/gcssink/gcs_sink.go +++ b/weed/replication/sink/gcssink/gcs_sink.go @@ -8,7 +8,7 @@ import ( "cloud.google.com/go/storage" "google.golang.org/api/option" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/replication/sink" @@ -89,8 +89,8 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error { return nil } - totalSize := filer2.FileSize(entry) - chunkViews := filer2.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) + totalSize := filer.FileSize(entry) + chunkViews := filer.ViewFromChunks(g.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) wc := g.client.Bucket(g.bucket).Object(key).NewWriter(context.Background()) diff --git a/weed/replication/sink/s3sink/s3_sink.go b/weed/replication/sink/s3sink/s3_sink.go index 56fc1930d..4e7df8ff2 100644 --- a/weed/replication/sink/s3sink/s3_sink.go +++ b/weed/replication/sink/s3sink/s3_sink.go @@ -12,7 +12,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/replication/sink" @@ -107,8 +107,8 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error { return err } - totalSize := filer2.FileSize(entry) - chunkViews := filer2.ViewFromChunks(s3sink.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) + totalSize := filer.FileSize(entry) + chunkViews := filer.ViewFromChunks(s3sink.filerSource.LookupFileId, entry.Chunks, 0, int64(totalSize)) parts := make([]*s3.CompletedPart, len(chunkViews)) @@ -116,7 +116,7 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry) error { for chunkIndex, chunk := range chunkViews { partId := chunkIndex + 1 wg.Add(1) - go func(chunk *filer2.ChunkView, index int) { + go func(chunk *filer.ChunkView, index int) { defer wg.Done() if part, uploadErr := s3sink.uploadPart(key, uploadId, partId, chunk); uploadErr != nil { err = uploadErr diff --git a/weed/replication/sink/s3sink/s3_write.go b/weed/replication/sink/s3sink/s3_write.go index c5c65ed5c..8a8e7a92b 100644 --- a/weed/replication/sink/s3sink/s3_write.go +++ b/weed/replication/sink/s3sink/s3_write.go @@ -9,7 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/s3" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -103,7 +103,7 @@ func (s3sink *S3Sink) completeMultipartUpload(ctx context.Context, key, uploadId } // To upload a part -func (s3sink *S3Sink) uploadPart(key, uploadId string, partId int, chunk *filer2.ChunkView) (*s3.CompletedPart, error) { +func (s3sink *S3Sink) uploadPart(key, uploadId string, partId int, chunk *filer.ChunkView) (*s3.CompletedPart, error) { var readSeeker io.ReadSeeker readSeeker, err := s3sink.buildReadSeeker(chunk) @@ -156,7 +156,7 @@ func (s3sink *S3Sink) uploadPartCopy(key, uploadId string, partId int64, copySou return err } -func (s3sink *S3Sink) buildReadSeeker(chunk *filer2.ChunkView) (io.ReadSeeker, error) { +func (s3sink *S3Sink) buildReadSeeker(chunk *filer.ChunkView) (io.ReadSeeker, error) { fileUrl, err := s3sink.filerSource.LookupFileId(chunk.FileId) if err != nil { return nil, err diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index a67a86454..4eb9bf32c 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -12,7 +12,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/google/uuid" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -108,7 +108,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{ Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)), Bucket: input.Bucket, - ETag: aws.String("\"" + filer2.ETagChunks(finalParts) + "\""), + ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""), Key: objectKey(input.Key), }, } @@ -208,8 +208,8 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP output.Parts = append(output.Parts, &s3.Part{ PartNumber: aws.Int64(int64(partNumber)), LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()), - Size: aws.Int64(int64(filer2.FileSize(entry))), - ETag: aws.String("\"" + filer2.ETag(entry) + "\""), + Size: aws.Int64(int64(filer.FileSize(entry))), + ETag: aws.String("\"" + filer.ETag(entry) + "\""), }) } } diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 254a99275..b6779dfb7 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -139,8 +139,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m contents = append(contents, ListEntry{ Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer2.ETag(entry) + "\"", - Size: int64(filer2.FileSize(entry)), + ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), Owner: CanonicalUser{ ID: fmt.Sprintf("%x", entry.Attributes.Uid), DisplayName: entry.Attributes.UserName, diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 1d39fdc76..d3ced0a53 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -8,7 +8,7 @@ import ( "strconv" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -34,7 +34,7 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L Entry: &filer_pb.Entry{ Name: req.Name, IsDirectory: entry.IsDirectory(), - Attributes: filer2.EntryAttributeToPb(entry), + Attributes: filer.EntryAttributeToPb(entry), Chunks: entry.Chunks, Extended: entry.Extended, }, @@ -50,7 +50,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file limit = fs.option.DirListingLimit } - paginationLimit := filer2.PaginationSize + paginationLimit := filer.PaginationSize if limit < paginationLimit { paginationLimit = limit } @@ -78,7 +78,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file Name: entry.Name(), IsDirectory: entry.IsDirectory(), Chunks: entry.Chunks, - Attributes: filer2.EntryAttributeToPb(entry), + Attributes: filer.EntryAttributeToPb(entry), Extended: entry.Extended, }, }); err != nil { @@ -160,9 +160,9 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr return } - createErr := fs.filer.CreateEntry(ctx, &filer2.Entry{ + createErr := fs.filer.CreateEntry(ctx, &filer.Entry{ FullPath: util.JoinPath(req.Directory, req.Entry.Name), - Attr: filer2.PbToEntryAttribute(req.Entry.Attributes), + Attr: filer.PbToEntryAttribute(req.Entry.Attributes), Chunks: chunks, }, req.OExcl, req.IsFromOtherCluster, req.Signatures) @@ -191,7 +191,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("UpdateEntry cleanupChunks %s: %v", fullpath, err2) } - newEntry := &filer2.Entry{ + newEntry := &filer.Entry{ FullPath: util.JoinPath(req.Directory, req.Entry.Name), Attr: entry.Attr, Extended: req.Entry.Extended, @@ -218,7 +218,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr } - if filer2.EqualEntry(entry, newEntry) { + if filer.EqualEntry(entry, newEntry) { return &filer_pb.UpdateEntryResponse{}, err } @@ -233,23 +233,23 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr return &filer_pb.UpdateEntryResponse{}, err } -func (fs *FilerServer) cleanupChunks(existingEntry *filer2.Entry, newEntry *filer_pb.Entry) (chunks, garbage []*filer_pb.FileChunk, err error) { +func (fs *FilerServer) cleanupChunks(existingEntry *filer.Entry, newEntry *filer_pb.Entry) (chunks, garbage []*filer_pb.FileChunk, err error) { // remove old chunks if not included in the new ones if existingEntry != nil { - garbage, err = filer2.MinusChunks(fs.lookupFileId, existingEntry.Chunks, newEntry.Chunks) + garbage, err = filer.MinusChunks(fs.lookupFileId, existingEntry.Chunks, newEntry.Chunks) if err != nil { return newEntry.Chunks, nil, fmt.Errorf("MinusChunks: %v", err) } } // files with manifest chunks are usually large and append only, skip calculating covered chunks - manifestChunks, nonManifestChunks := filer2.SeparateManifestChunks(newEntry.Chunks) + manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(newEntry.Chunks) - chunks, coveredChunks := filer2.CompactFileChunks(fs.lookupFileId, nonManifestChunks) + chunks, coveredChunks := filer.CompactFileChunks(fs.lookupFileId, nonManifestChunks) garbage = append(garbage, coveredChunks...) - chunks, err = filer2.MaybeManifestize(fs.saveAsChunk( + chunks, err = filer.MaybeManifestize(fs.saveAsChunk( newEntry.Attributes.Replication, newEntry.Attributes.Collection, "", @@ -273,9 +273,9 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo var offset int64 = 0 entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath)) if err == filer_pb.ErrNotFound { - entry = &filer2.Entry{ + entry = &filer.Entry{ FullPath: fullpath, - Attr: filer2.Attr{ + Attr: filer.Attr{ Crtime: time.Now(), Mtime: time.Now(), Mode: os.FileMode(0644), @@ -284,7 +284,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo }, } } else { - offset = int64(filer2.TotalSize(entry.Chunks)) + offset = int64(filer.TotalSize(entry.Chunks)) } for _, chunk := range req.Chunks { @@ -294,7 +294,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo entry.Chunks = append(entry.Chunks, req.Chunks...) - entry.Chunks, err = filer2.MaybeManifestize(fs.saveAsChunk( + entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk( entry.Replication, entry.Collection, "", diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index e6eef35cb..35df01665 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -5,7 +5,7 @@ import ( "fmt" "path/filepath" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -43,7 +43,7 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom return &filer_pb.AtomicRenameEntryResponse{}, nil } -func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer2.Entry, newParent util.FullPath, newName string, events *MoveEvents) error { +func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error { if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, events, func() error { if entry.IsDirectory() { @@ -59,7 +59,7 @@ func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, e return nil } -func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer2.Entry, newParent util.FullPath, newName string, events *MoveEvents) error { +func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error { currentDirPath := oldParent.Child(entry.Name()) newDirPath := newParent.Child(newName) @@ -92,7 +92,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. return nil } -func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer2.Entry, newParent util.FullPath, newName string, events *MoveEvents, +func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents, moveFolderSubEntries func() error) error { oldPath, newPath := oldParent.Child(entry.Name()), newParent.Child(newName) @@ -105,7 +105,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat } // add to new directory - newEntry := &filer2.Entry{ + newEntry := &filer.Entry{ FullPath: newPath, Attr: entry.Attr, Chunks: entry.Chunks, @@ -136,6 +136,6 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat } type MoveEvents struct { - oldEntries []*filer2.Entry - newEntries []*filer2.Entry + oldEntries []*filer.Entry + newEntries []*filer.Entry } diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index 2ad12e9c8..9ba45edfe 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -7,7 +7,7 @@ import ( "github.com/golang/protobuf/proto" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -63,7 +63,7 @@ func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq eachLogEntryFn := eachLogEntryFn(eachEventNotificationFn) - if _, ok := fs.filer.Store.ActualStore.(filer2.FilerLocalStore); ok { + if _, ok := fs.filer.Store.ActualStore.(filer.FilerLocalStore); ok { // println("reading from persisted logs ...") processedTsNs, err := fs.filer.ReadPersistedLogBuffer(lastReadTime, eachLogEntryFn) if err != nil { @@ -124,7 +124,7 @@ func eachEventNotificationFn(req *filer_pb.SubscribeMetadataRequest, stream file fullpath := util.Join(dirPath, entryName) // skip on filer internal meta logs - if strings.HasPrefix(fullpath, filer2.SystemLogDir) { + if strings.HasPrefix(fullpath, filer.SystemLogDir) { return nil } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 6995c7cfe..160ea5a6d 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -18,16 +18,16 @@ import ( "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/filer2" - _ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra" - _ "github.com/chrislusf/seaweedfs/weed/filer2/etcd" - _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" - _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb2" - _ "github.com/chrislusf/seaweedfs/weed/filer2/mongodb" - _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" - _ "github.com/chrislusf/seaweedfs/weed/filer2/postgres" - _ "github.com/chrislusf/seaweedfs/weed/filer2/redis" - _ "github.com/chrislusf/seaweedfs/weed/filer2/redis2" + "github.com/chrislusf/seaweedfs/weed/filer" + _ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" + _ "github.com/chrislusf/seaweedfs/weed/filer/etcd" + _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb" + _ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2" + _ "github.com/chrislusf/seaweedfs/weed/filer/mongodb" + _ "github.com/chrislusf/seaweedfs/weed/filer/mysql" + _ "github.com/chrislusf/seaweedfs/weed/filer/postgres" + _ "github.com/chrislusf/seaweedfs/weed/filer/redis" + _ "github.com/chrislusf/seaweedfs/weed/filer/redis2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/notification" _ "github.com/chrislusf/seaweedfs/weed/notification/aws_sqs" @@ -58,7 +58,7 @@ type FilerOption struct { type FilerServer struct { option *FilerOption secret security.SigningKey - filer *filer2.Filer + filer *filer.Filer grpcDialOption grpc.DialOption // notifying clients @@ -82,7 +82,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) glog.Fatal("master list is required!") } - fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.Port, option.Collection, option.DefaultReplication, func() { + fs.filer = filer.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.Port, option.Collection, option.DefaultReplication, func() { fs.listenersCond.Broadcast() }) fs.filer.Cipher = option.Cipher diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 449b9f1a0..fbd45d6b9 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -94,7 +94,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } // set etag - etag := filer2.ETagEntry(entry) + etag := filer.ETagEntry(entry) if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" { w.WriteHeader(http.StatusNotModified) return @@ -115,7 +115,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, ext := filepath.Ext(filename) width, height, mode, shouldResize := shouldResizeImages(ext, r) if shouldResize { - data, err := filer2.ReadAll(fs.filer.MasterClient, entry.Chunks) + data, err := filer.ReadAll(fs.filer.MasterClient, entry.Chunks) if err != nil { glog.Errorf("failed to read %s: %v", path, err) w.WriteHeader(http.StatusNotModified) @@ -128,7 +128,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { - return filer2.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size) + return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size) }) } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 1d037f85f..0f6176356 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -86,7 +86,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite return nil, nil, err } - fileChunks, replyerr = filer2.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, ttlString, fsync), fileChunks) + fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, ttlString, fsync), fileChunks) if replyerr != nil { glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) return @@ -108,7 +108,7 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter return nil, nil, err } - fileChunks, replyerr = filer2.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, ttlString, fsync), fileChunks) + fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, ttlString, fsync), fileChunks) if replyerr != nil { glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr) return @@ -149,9 +149,9 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } glog.V(4).Infoln("saving", path) - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: util.FullPath(path), - Attr: filer2.Attr{ + Attr: filer.Attr{ Mtime: time.Now(), Crtime: crTime, Mode: os.FileMode(mode), @@ -236,7 +236,7 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht return uploadResult, err } -func (fs *FilerServer) saveAsChunk(replication string, collection string, dataCenter string, ttlString string, fsync bool) filer2.SaveDataAsChunkFunctionType { +func (fs *FilerServer) saveAsChunk(replication string, collection string, dataCenter string, ttlString string, fsync bool) filer.SaveDataAsChunkFunctionType { return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) { // assign one file id for one chunk diff --git a/weed/server/filer_server_handlers_write_cipher.go b/weed/server/filer_server_handlers_write_cipher.go index 670399425..60082a8d4 100644 --- a/weed/server/filer_server_handlers_write_cipher.go +++ b/weed/server/filer_server_handlers_write_cipher.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -58,9 +58,9 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht } } - entry := &filer2.Entry{ + entry := &filer.Entry{ FullPath: util.FullPath(path), - Attr: filer2.Attr{ + Attr: filer.Attr{ Mtime: time.Now(), Crtime: time.Now(), Mode: 0660, diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index f06189e34..57723ab0b 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -19,7 +19,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util/chunk_cache" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" ) @@ -41,7 +41,7 @@ type WebDavOption struct { type WebDavServer struct { option *WebDavOption secret security.SigningKey - filer *filer2.Filer + filer *filer.Filer grpcDialOption grpc.DialOption Handler *webdav.Handler } @@ -67,7 +67,7 @@ func NewWebDavServer(option *WebDavOption) (ws *WebDavServer, err error) { type WebDavFileSystem struct { option *WebDavOption secret security.SigningKey - filer *filer2.Filer + filer *filer.Filer grpcDialOption grpc.DialOption chunkCache *chunk_cache.TieredChunkCache signature int32 @@ -94,7 +94,7 @@ type WebDavFile struct { isDirectory bool off int64 entry *filer_pb.Entry - entryViewCache []filer2.VisibleInterval + entryViewCache []filer.VisibleInterval reader io.ReaderAt } @@ -338,7 +338,7 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F if err != nil { return nil, err } - fi.size = int64(filer2.FileSize(entry)) + fi.size = int64(filer.FileSize(entry)) fi.name = string(fullpath) fi.mode = os.FileMode(entry.Attributes.FileMode) fi.modifiledTime = time.Unix(entry.Attributes.Mtime, 0) @@ -471,17 +471,17 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) { if err != nil { return 0, err } - fileSize := int64(filer2.FileSize(f.entry)) + fileSize := int64(filer.FileSize(f.entry)) if fileSize == 0 { return 0, io.EOF } if f.entryViewCache == nil { - f.entryViewCache, _ = filer2.NonOverlappingVisibleIntervals(filer2.LookupFn(f.fs), f.entry.Chunks) + f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.Chunks) f.reader = nil } if f.reader == nil { - chunkViews := filer2.ViewFromVisibleIntervals(f.entryViewCache, 0, math.MaxInt32) - f.reader = filer2.NewChunkReaderAtFromClient(f.fs, chunkViews, f.fs.chunkCache, fileSize) + chunkViews := filer.ViewFromVisibleIntervals(f.entryViewCache, 0, math.MaxInt32) + f.reader = filer.NewChunkReaderAtFromClient(f.fs, chunkViews, f.fs.chunkCache, fileSize) } readSize, err = f.reader.ReadAt(p, f.off) @@ -509,7 +509,7 @@ func (f *WebDavFile) Readdir(count int) (ret []os.FileInfo, err error) { err = filer_pb.ReadDirAllEntries(f.fs, util.FullPath(dir), "", func(entry *filer_pb.Entry, isLast bool) error { fi := FileInfo{ - size: int64(filer2.FileSize(entry)), + size: int64(filer.FileSize(entry)), name: entry.Name, mode: os.FileMode(entry.Attributes.FileMode), modifiledTime: time.Unix(entry.Attributes.Mtime, 0), diff --git a/weed/shell/command_fs_cat.go b/weed/shell/command_fs_cat.go index 7177d8ac3..3c5e13663 100644 --- a/weed/shell/command_fs_cat.go +++ b/weed/shell/command_fs_cat.go @@ -5,7 +5,7 @@ import ( "io" "math" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -52,7 +52,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write return err } - return filer2.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) }) diff --git a/weed/shell/command_fs_du.go b/weed/shell/command_fs_du.go index 5404b0cdb..71003714d 100644 --- a/weed/shell/command_fs_du.go +++ b/weed/shell/command_fs_du.go @@ -4,7 +4,7 @@ import ( "fmt" "io" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -70,7 +70,7 @@ func duTraverseDirectory(writer io.Writer, filerClient filer_pb.FilerClient, dir } } else { fileBlockCount = uint64(len(entry.Chunks)) - fileByteCount = filer2.FileSize(entry) + fileByteCount = filer.FileSize(entry) blockCount += fileBlockCount byteCount += fileByteCount } diff --git a/weed/shell/command_fs_ls.go b/weed/shell/command_fs_ls.go index 4110c7b8d..592ec8be0 100644 --- a/weed/shell/command_fs_ls.go +++ b/weed/shell/command_fs_ls.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -95,7 +95,7 @@ func (c *commandFsLs) Do(args []string, commandEnv *CommandEnv, writer io.Writer fmt.Fprintf(writer, "%s %3d %s %s %6d %s/%s\n", fileMode, len(entry.Chunks), userName, groupName, - filer2.FileSize(entry), dir, entry.Name) + filer.FileSize(entry), dir, entry.Name) } else { fmt.Fprintf(writer, "%s\n", entry.Name) } diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index cf5ad6d6d..4b3568acb 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -11,7 +11,7 @@ import ( "path/filepath" "sync" - "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -197,7 +197,7 @@ func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToSer files[i.vid].Write(buffer) } }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { - dChunks, mChunks, resolveErr := filer2.ResolveChunkManifest(filer2.LookupFn(c.env), entry.Entry.Chunks) + dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks) if resolveErr != nil { return nil } From d91ec535b32efe645206ad64ccae8e88c2bf6c6e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 01:29:13 -0700 Subject: [PATCH 24/38] fix tests --- weed/filer/leveldb/leveldb_store_test.go | 4 ++-- weed/filer/leveldb2/leveldb2_store_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index df196b02e..d656c690a 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -6,7 +6,7 @@ import ( "os" "testing" - "github.com/chrislusf/seaweedfs/weed/filer" + filer2 "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -22,7 +22,7 @@ func TestCreateAndFind(t *testing.T) { ctx := context.Background() - entry1 := &filer.Entry{ + entry1 := &filer2.Entry{ FullPath: fullpath, Attr: filer.Attr{ Mode: 0440, diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index 191de0040..8939720ee 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -6,7 +6,7 @@ import ( "os" "testing" - "github.com/chrislusf/seaweedfs/weed/filer" + filer2 "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -22,7 +22,7 @@ func TestCreateAndFind(t *testing.T) { ctx := context.Background() - entry1 := &filer.Entry{ + entry1 := &filer2.Entry{ FullPath: fullpath, Attr: filer.Attr{ Mode: 0440, From 2b14ae581916f324f8cc6fdf4778d948f9f3b6fd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 01:33:43 -0700 Subject: [PATCH 25/38] fix tests --- weed/filer/leveldb/leveldb_store_test.go | 22 +++++++++++----------- weed/filer/leveldb2/leveldb2_store_test.go | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index d656c690a..b07f81129 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -6,23 +6,23 @@ import ( "os" "testing" - filer2 "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" ) func TestCreateAndFind(t *testing.T) { - filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") defer os.RemoveAll(dir) store := &LevelDBStore{} store.initialize(dir) - filer.SetStore(store) + testFiler.SetStore(store) fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg") ctx := context.Background() - entry1 := &filer2.Entry{ + entry1 := &filer.Entry{ FullPath: fullpath, Attr: filer.Attr{ Mode: 0440, @@ -31,12 +31,12 @@ func TestCreateAndFind(t *testing.T) { }, } - if err := filer.CreateEntry(ctx, entry1, false, false, nil); err != nil { + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { t.Errorf("create entry %v: %v", entry1.FullPath, err) return } - entry, err := filer.FindEntry(ctx, fullpath) + entry, err := testFiler.FindEntry(ctx, fullpath) if err != nil { t.Errorf("find entry: %v", err) @@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") + entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -65,17 +65,17 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") defer os.RemoveAll(dir) store := &LevelDBStore{} store.initialize(dir) - filer.SetStore(store) + testFiler.SetStore(store) ctx := context.Background() // checking one upper directory - entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index 8939720ee..c9b140951 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -6,23 +6,23 @@ import ( "os" "testing" - filer2 "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/util" ) func TestCreateAndFind(t *testing.T) { - filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") defer os.RemoveAll(dir) store := &LevelDB2Store{} store.initialize(dir, 2) - filer.SetStore(store) + testFiler.SetStore(store) fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg") ctx := context.Background() - entry1 := &filer2.Entry{ + entry1 := &filer.Entry{ FullPath: fullpath, Attr: filer.Attr{ Mode: 0440, @@ -31,12 +31,12 @@ func TestCreateAndFind(t *testing.T) { }, } - if err := filer.CreateEntry(ctx, entry1, false, false, nil); err != nil { + if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil { t.Errorf("create entry %v: %v", entry1.FullPath, err) return } - entry, err := filer.FindEntry(ctx, fullpath) + entry, err := testFiler.FindEntry(ctx, fullpath) if err != nil { t.Errorf("find entry: %v", err) @@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") + entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -65,17 +65,17 @@ func TestCreateAndFind(t *testing.T) { } func TestEmptyRoot(t *testing.T) { - filer := filer.NewFiler(nil, nil, "", 0, "", "", nil) + testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2") defer os.RemoveAll(dir) store := &LevelDB2Store{} store.initialize(dir, 2) - filer.SetStore(store) + testFiler.SetStore(store) ctx := context.Background() // checking one upper directory - entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") + entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if err != nil { t.Errorf("list entries: %v", err) return From 8e54e345760c2dca12085311d6fe0cf7eba8b6a9 Mon Sep 17 00:00:00 2001 From: James Hartig Date: Tue, 1 Sep 2020 22:00:00 -0400 Subject: [PATCH 26/38] volume: Don't unmount before deleting volume in copy If we unmount first and then delete, the delete fails because the volume was unmounted. Delete ends up doing the same thing as the unmount anyways. --- weed/server/volume_grpc_copy.go | 9 ++------- weed/storage/disk_location.go | 3 +++ weed/storage/store.go | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go index 5c7d5572c..cd2b53c8a 100644 --- a/weed/server/volume_grpc_copy.go +++ b/weed/server/volume_grpc_copy.go @@ -27,17 +27,12 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo glog.V(0).Infof("volume %d already exists. deleted before copying...", req.VolumeId) - err := vs.store.UnmountVolume(needle.VolumeId(req.VolumeId)) - if err != nil { - return nil, fmt.Errorf("failed to mount existing volume %d: %v", req.VolumeId, err) - } - - err = vs.store.DeleteVolume(needle.VolumeId(req.VolumeId)) + err := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId)) if err != nil { return nil, fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err) } - glog.V(0).Infof("deleted exisitng volume %d before copying.", req.VolumeId) + glog.V(0).Infof("deleted existing volume %d before copying.", req.VolumeId) } location := vs.store.FindFreeLocation() diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index c309b3f92..9ecc57459 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -174,6 +174,9 @@ func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e er } func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) { + l.volumesLock.Lock() + defer l.volumesLock.Unlock() + v, ok := l.volumes[vid] if !ok { return diff --git a/weed/storage/store.go b/weed/storage/store.go index 3f16688bf..48cbeb3d1 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -380,7 +380,7 @@ func (s *Store) DeleteVolume(i needle.VolumeId) error { Ttl: v.Ttl.ToUint32(), } for _, location := range s.Locations { - if found, error := location.deleteVolumeById(i); found && error == nil { + if found, err := location.deleteVolumeById(i); found && err == nil { glog.V(0).Infof("DeleteVolume %d", i) s.DeletedVolumesChan <- message return nil From 37234bf3f894edd727dc0abf8fff001b95af5a2e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 21:58:57 -0700 Subject: [PATCH 27/38] filer store adds kv support can compile now, need to implement those unimplemented --- .../abstract_sql/abstract_sql_store_kv.go | 18 ++++++ weed/filer/cassandra/cassandra_store_kv.go | 18 ++++++ weed/filer/etcd/etcd_store_kv.go | 44 +++++++++++++++ weed/filer/filer.go | 2 - weed/filer/filerstore.go | 21 +++++++ weed/filer/leveldb/leveldb_store_kv.go | 39 +++++++++++++ weed/filer/leveldb2/leveldb2_store_kv.go | 56 +++++++++++++++++++ weed/filer/mongodb/mongodb_store_kv.go | 19 +++++++ weed/filer/redis/universal_redis_store_kv.go | 42 ++++++++++++++ weed/filer/redis2/universal_redis_store_kv.go | 42 ++++++++++++++ 10 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 weed/filer/abstract_sql/abstract_sql_store_kv.go create mode 100644 weed/filer/cassandra/cassandra_store_kv.go create mode 100644 weed/filer/etcd/etcd_store_kv.go create mode 100644 weed/filer/leveldb/leveldb_store_kv.go create mode 100644 weed/filer/leveldb2/leveldb2_store_kv.go create mode 100644 weed/filer/mongodb/mongodb_store_kv.go create mode 100644 weed/filer/redis/universal_redis_store_kv.go create mode 100644 weed/filer/redis2/universal_redis_store_kv.go diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go new file mode 100644 index 000000000..cfffc8918 --- /dev/null +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -0,0 +1,18 @@ +package abstract_sql + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" +) + +func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + return filer.ErrKvNotImplemented +} + +func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + return nil, filer.ErrKvNotImplemented +} + +func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err error) { + return filer.ErrKvNotImplemented +} diff --git a/weed/filer/cassandra/cassandra_store_kv.go b/weed/filer/cassandra/cassandra_store_kv.go new file mode 100644 index 000000000..f7668746f --- /dev/null +++ b/weed/filer/cassandra/cassandra_store_kv.go @@ -0,0 +1,18 @@ +package cassandra + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" +) + +func (store *CassandraStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + return filer.ErrKvNotImplemented +} + +func (store *CassandraStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + return nil, filer.ErrKvNotImplemented +} + +func (store *CassandraStore) KvDelete(ctx context.Context, key []byte) (err error) { + return filer.ErrKvNotImplemented +} diff --git a/weed/filer/etcd/etcd_store_kv.go b/weed/filer/etcd/etcd_store_kv.go new file mode 100644 index 000000000..a803a5834 --- /dev/null +++ b/weed/filer/etcd/etcd_store_kv.go @@ -0,0 +1,44 @@ +package etcd + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" +) + +func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + _, err = store.client.Put(ctx, string(key), string(value)) + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + resp, err := store.client.Get(ctx, string(key), nil) + + if err != nil { + return nil, fmt.Errorf("kv get: %v", err) + } + + if len(resp.Kvs) == 0 { + return nil, filer.ErrKvNotFound + } + + return resp.Kvs[0].Value, nil +} + +func (store *EtcdStore) KvDelete(ctx context.Context, key []byte) (err error) { + + _, err = store.client.Delete(ctx, string(key)) + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 71da1a080..16e86da01 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -2,7 +2,6 @@ package filer import ( "context" - "errors" "fmt" "os" "strings" @@ -25,7 +24,6 @@ const ( var ( OS_UID = uint32(os.Getuid()) OS_GID = uint32(os.Getgid()) - ErrUnsupportedListDirectoryPrefixed = errors.New("UNSUPPORTED") ) type Filer struct { diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index 48f7c99e4..518212437 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -2,6 +2,7 @@ package filer import ( "context" + "errors" "strings" "time" @@ -10,6 +11,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) +var ( + ErrUnsupportedListDirectoryPrefixed = errors.New("unsupported directory prefix listing") + ErrKvNotImplemented = errors.New("kv not implemented yet") + ErrKvNotFound = errors.New("kv: not found") +) + type FilerStore interface { // GetName gets the name to locate the configuration in filer.toml file GetName() string @@ -28,6 +35,10 @@ type FilerStore interface { CommitTransaction(ctx context.Context) error RollbackTransaction(ctx context.Context) error + KvPut(ctx context.Context, key []byte, value []byte) (err error) + KvGet(ctx context.Context, key []byte) (value []byte, err error) + KvDelete(ctx context.Context, key []byte) (err error) + Shutdown() } @@ -206,3 +217,13 @@ func (fsw *FilerStoreWrapper) RollbackTransaction(ctx context.Context) error { func (fsw *FilerStoreWrapper) Shutdown() { fsw.ActualStore.Shutdown() } + +func (fsw *FilerStoreWrapper) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + return fsw.ActualStore.KvPut(ctx, key, value) +} +func (fsw *FilerStoreWrapper) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + return fsw.ActualStore.KvGet(ctx, key) +} +func (fsw *FilerStoreWrapper) KvDelete(ctx context.Context, key []byte) (err error) { + return fsw.ActualStore.KvDelete(ctx, key) +} diff --git a/weed/filer/leveldb/leveldb_store_kv.go b/weed/filer/leveldb/leveldb_store_kv.go new file mode 100644 index 000000000..7fe1d3356 --- /dev/null +++ b/weed/filer/leveldb/leveldb_store_kv.go @@ -0,0 +1,39 @@ +package leveldb + +import ( + "context" + "fmt" +) + +func (store *LevelDBStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + err = store.db.Put(key, value, nil) + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *LevelDBStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + value, err = store.db.Get(key, nil) + + if err != nil { + return nil, fmt.Errorf("kv get: %v", err) + } + + return +} + +func (store *LevelDBStore) KvDelete(ctx context.Context, key []byte) (err error) { + + err = store.db.Delete(key, nil) + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/weed/filer/leveldb2/leveldb2_store_kv.go b/weed/filer/leveldb2/leveldb2_store_kv.go new file mode 100644 index 000000000..b415d3c32 --- /dev/null +++ b/weed/filer/leveldb2/leveldb2_store_kv.go @@ -0,0 +1,56 @@ +package leveldb + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/syndtr/goleveldb/leveldb" +) + +func (store *LevelDB2Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + partitionId := bucketKvKey(key, store.dbCount) + + err = store.dbs[partitionId].Put(key, value, nil) + + if err != nil { + return fmt.Errorf("kv bucket %d put: %v", partitionId, err) + } + + return nil +} + +func (store *LevelDB2Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + partitionId := bucketKvKey(key, store.dbCount) + + value, err = store.dbs[partitionId].Get(key, nil) + + if err == leveldb.ErrNotFound { + return nil, filer.ErrKvNotFound + } + + if err != nil { + return nil, fmt.Errorf("kv bucket %d get: %v", partitionId, err) + } + + return +} + +func (store *LevelDB2Store) KvDelete(ctx context.Context, key []byte) (err error) { + + partitionId := bucketKvKey(key, store.dbCount) + + err = store.dbs[partitionId].Delete(key, nil) + + if err != nil { + return fmt.Errorf("kv bucket %d delete: %v", partitionId, err) + } + + return nil +} + +func bucketKvKey(key []byte, dbCount int) (partitionId int) { + return int(key[len(key)-1]) % dbCount +} diff --git a/weed/filer/mongodb/mongodb_store_kv.go b/weed/filer/mongodb/mongodb_store_kv.go new file mode 100644 index 000000000..4e0a4b0f0 --- /dev/null +++ b/weed/filer/mongodb/mongodb_store_kv.go @@ -0,0 +1,19 @@ +package mongodb + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" +) + + +func (store *MongodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + return filer.ErrKvNotImplemented +} + +func (store *MongodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + return nil, filer.ErrKvNotImplemented +} + +func (store *MongodbStore) KvDelete(ctx context.Context, key []byte) (err error) { + return filer.ErrKvNotImplemented +} diff --git a/weed/filer/redis/universal_redis_store_kv.go b/weed/filer/redis/universal_redis_store_kv.go new file mode 100644 index 000000000..0fc12c631 --- /dev/null +++ b/weed/filer/redis/universal_redis_store_kv.go @@ -0,0 +1,42 @@ +package redis + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/go-redis/redis" +) + +func (store *UniversalRedisStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + _, err = store.Client.Set(string(key), value, 0).Result() + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *UniversalRedisStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + data, err := store.Client.Get(string(key)).Result() + + if err == redis.Nil { + return nil, filer.ErrKvNotFound + } + + return []byte(data), err +} + +func (store *UniversalRedisStore) KvDelete(ctx context.Context, key []byte) (err error) { + + _, err = store.Client.Del(string(key)).Result() + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} diff --git a/weed/filer/redis2/universal_redis_store_kv.go b/weed/filer/redis2/universal_redis_store_kv.go new file mode 100644 index 000000000..658491ddf --- /dev/null +++ b/weed/filer/redis2/universal_redis_store_kv.go @@ -0,0 +1,42 @@ +package redis2 + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/go-redis/redis" +) + +func (store *UniversalRedis2Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) { + + _, err = store.Client.Set(string(key), value, 0).Result() + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil +} + +func (store *UniversalRedis2Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) { + + data, err := store.Client.Get(string(key)).Result() + + if err == redis.Nil { + return nil, filer.ErrKvNotFound + } + + return []byte(data), err +} + +func (store *UniversalRedis2Store) KvDelete(ctx context.Context, key []byte) (err error) { + + _, err = store.Client.Del(string(key)).Result() + + if err != nil { + return fmt.Errorf("kv delete: %v", err) + } + + return nil +} From 2815bbe6c06612d6c7ffb60b729520464e6bf3ba Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 21:59:26 -0700 Subject: [PATCH 28/38] go fmt --- weed/filer/filer.go | 4 ++-- weed/filer/mongodb/mongodb_store_kv.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 16e86da01..7a555372f 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -22,8 +22,8 @@ const ( ) var ( - OS_UID = uint32(os.Getuid()) - OS_GID = uint32(os.Getgid()) + OS_UID = uint32(os.Getuid()) + OS_GID = uint32(os.Getgid()) ) type Filer struct { diff --git a/weed/filer/mongodb/mongodb_store_kv.go b/weed/filer/mongodb/mongodb_store_kv.go index 4e0a4b0f0..e1f5f7fdf 100644 --- a/weed/filer/mongodb/mongodb_store_kv.go +++ b/weed/filer/mongodb/mongodb_store_kv.go @@ -5,7 +5,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer" ) - func (store *MongodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { return filer.ErrKvNotImplemented } From 72b0a5f1d1481d779be64f24ab8cc3fad056847c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 22:25:17 -0700 Subject: [PATCH 29/38] mysql or postgres: log find error --- weed/filer/abstract_sql/abstract_sql_store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index a6de2ea39..891fe305f 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -118,6 +118,7 @@ func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.Full row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) var data []byte if err := row.Scan(&data); err != nil { + glog.Errorf("find %s: %v", fullpath, err) return nil, filer_pb.ErrNotFound } From 87d7312bf6edea56c2529b5be39e6ca033909840 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 22:47:57 -0700 Subject: [PATCH 30/38] mysql/postgres: properly report entry not found --- weed/filer/abstract_sql/abstract_sql_store.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 891fe305f..e87ecb9fb 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -116,10 +116,13 @@ func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.Full dir, name := fullpath.DirAndName() row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir) + var data []byte if err := row.Scan(&data); err != nil { - glog.Errorf("find %s: %v", fullpath, err) - return nil, filer_pb.ErrNotFound + if err == sql.ErrNoRows { + return nil, filer_pb.ErrNotFound + } + return nil, fmt.Errorf("find %s: %v", fullpath, err) } entry := &filer.Entry{ From aa40295f06b198e873b84c6f304c071b92a6b375 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 22:48:23 -0700 Subject: [PATCH 31/38] leveldb: report not found entry --- weed/filer/leveldb/leveldb_store_kv.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/weed/filer/leveldb/leveldb_store_kv.go b/weed/filer/leveldb/leveldb_store_kv.go index 7fe1d3356..f686cbf21 100644 --- a/weed/filer/leveldb/leveldb_store_kv.go +++ b/weed/filer/leveldb/leveldb_store_kv.go @@ -3,6 +3,8 @@ package leveldb import ( "context" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/syndtr/goleveldb/leveldb" ) func (store *LevelDBStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { @@ -20,6 +22,10 @@ func (store *LevelDBStore) KvGet(ctx context.Context, key []byte) (value []byte, value, err = store.db.Get(key, nil) + if err == leveldb.ErrNotFound { + return nil, filer.ErrKvNotFound + } + if err != nil { return nil, fmt.Errorf("kv get: %v", err) } From 06a118826746c0a8b69bbf4069f2c9b9b4afa37d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 22:48:36 -0700 Subject: [PATCH 32/38] mysql/postgres: support kv operations --- .../abstract_sql/abstract_sql_store_kv.go | 78 ++++++++++++++++++- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index cfffc8918..de7d7b7a5 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -2,17 +2,89 @@ package abstract_sql import ( "context" + "database/sql" + "fmt" "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" ) func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - return filer.ErrKvNotImplemented + + dirStr, dirHash, name := genDirAndName(key) + + res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value) + if err != nil { + return fmt.Errorf("kv insert: %s", err) + } + + // TODO maybe it will throw error before coming here? + + affectedRows, err := res.RowsAffected() + if err == nil && affectedRows > 0 { + return nil + } + + // now the insert failed possibly due to duplication constraints + glog.V(1).Infof("kv insert falls back to update: %s", err) + + res, err = store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, value, dirHash, name, dirStr) + if err != nil { + return fmt.Errorf("kv upsert: %s", err) + } + + _, err = res.RowsAffected() + if err != nil { + return fmt.Errorf("kv upsert no rows affected: %s", err) + } + return nil + } func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - return nil, filer.ErrKvNotImplemented + + dirStr, dirHash, name := genDirAndName(key) + row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, dirHash, name, dirStr) + + err = row.Scan(&value) + + if err == sql.ErrNoRows { + return nil, filer.ErrKvNotFound + } + + if err != nil { + return nil, fmt.Errorf("kv get: %v", err) + } + + return } func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err error) { - return filer.ErrKvNotImplemented + + dirStr, dirHash, name := genDirAndName(key) + + res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, dirHash, name, dirStr) + if err != nil { + return fmt.Errorf("kv delete: %s", err) + } + + _, err = res.RowsAffected() + if err != nil { + return fmt.Errorf("kv delete %s but no rows affected: %s", err) + } + + return nil + } + +func genDirAndName(key []byte) (dirStr string, dirHash int64, name string) { + for len(key) < 8 { + key = append(key, 0) + } + + dirHash = int64(util.BytesToUint64(key[:8])) + dirStr = string(key[:8]) + name = string(key[8:]) + + return +} \ No newline at end of file From 9ea290aa123f852b22ff7bbf1a0a9227f36f696b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 1 Sep 2020 23:21:41 -0700 Subject: [PATCH 33/38] fix error message --- weed/filer/abstract_sql/abstract_sql_store_kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index de7d7b7a5..5084b6062 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -70,7 +70,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er _, err = res.RowsAffected() if err != nil { - return fmt.Errorf("kv delete %s but no rows affected: %s", err) + return fmt.Errorf("kv delete no rows affected: %s", err) } return nil From ed62f524524f557c9319f2199b2f3299866969cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 2 Sep 2020 10:09:49 -0700 Subject: [PATCH 34/38] fix sql insert with duplicated primary key --- weed/filer/abstract_sql/abstract_sql_store.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index e87ecb9fb..368bec973 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" + "strings" ) type AbstractSqlStore struct { @@ -68,12 +69,9 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta) if err != nil { - return fmt.Errorf("insert %s: %s", entry.FullPath, err) - } - - affectedRows, err := res.RowsAffected() - if err == nil && affectedRows > 0 { - return nil + if !strings.Contains(strings.ToLower(err.Error()), "duplicate") { + return fmt.Errorf("kv insert: %s", err) + } } // now the insert failed possibly due to duplication constraints From 05a6e2dc7f88131fd7f2e99ff283b4712a623cd3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 2 Sep 2020 10:12:44 -0700 Subject: [PATCH 35/38] sql kv upsert --- weed/filer/abstract_sql/abstract_sql_store_kv.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index 5084b6062..ace7c4e08 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -7,6 +7,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/util" + "strings" ) func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { @@ -15,14 +16,9 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, dirHash, name, dirStr, value) if err != nil { - return fmt.Errorf("kv insert: %s", err) - } - - // TODO maybe it will throw error before coming here? - - affectedRows, err := res.RowsAffected() - if err == nil && affectedRows > 0 { - return nil + if !strings.Contains(strings.ToLower(err.Error()), "duplicate") { + return fmt.Errorf("kv insert: %s", err) + } } // now the insert failed possibly due to duplication constraints @@ -87,4 +83,4 @@ func genDirAndName(key []byte) (dirStr string, dirHash int64, name string) { name = string(key[8:]) return -} \ No newline at end of file +} From 7c770b727cc7f102b14ca8eac4c2e8af458b50d7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 2 Sep 2020 10:13:48 -0700 Subject: [PATCH 36/38] Update abstract_sql_store_kv.go --- weed/filer/abstract_sql/abstract_sql_store_kv.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weed/filer/abstract_sql/abstract_sql_store_kv.go b/weed/filer/abstract_sql/abstract_sql_store_kv.go index ace7c4e08..b5a662c6b 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_kv.go +++ b/weed/filer/abstract_sql/abstract_sql_store_kv.go @@ -4,10 +4,11 @@ import ( "context" "database/sql" "fmt" + "strings" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/util" - "strings" ) func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { From ecaa30c4087b2d43b6c25431ac7842a771e6f3ee Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 2 Sep 2020 17:17:44 -0700 Subject: [PATCH 37/38] better error message --- weed/filer/mongodb/mongodb_store.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index 57d9a031d..104d1f9e2 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -109,6 +109,10 @@ func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) Meta: meta, }) + if err != nil { + return fmt.Errorf("InsertEntry %st: %v", entry.FullPath, err) + } + return nil } @@ -124,6 +128,7 @@ func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath var where = bson.M{"directory": dir, "name": name} err = store.connect.Database(store.database).Collection(store.collectionName).FindOne(ctx, where).Decode(&data) if err != mongo.ErrNoDocuments && err != nil { + glog.Error("find %s: %v", fullpath, err) return nil, filer_pb.ErrNotFound } From 645a4af3db6fa97440fcaca632bb1ddd939f8268 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 2 Sep 2020 17:19:14 -0700 Subject: [PATCH 38/38] mongodb: support kv operations --- weed/filer/mongodb/mongodb_store_kv.go | 60 ++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/weed/filer/mongodb/mongodb_store_kv.go b/weed/filer/mongodb/mongodb_store_kv.go index e1f5f7fdf..09508e691 100644 --- a/weed/filer/mongodb/mongodb_store_kv.go +++ b/weed/filer/mongodb/mongodb_store_kv.go @@ -2,17 +2,71 @@ package mongodb import ( "context" + "fmt" "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" ) func (store *MongodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { - return filer.ErrKvNotImplemented + + dir, name := genDirAndName(key) + + c := store.connect.Database(store.database).Collection(store.collectionName) + + _, err = c.InsertOne(ctx, Model{ + Directory: dir, + Name: name, + Meta: value, + }) + + if err != nil { + return fmt.Errorf("kv put: %v", err) + } + + return nil } func (store *MongodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { - return nil, filer.ErrKvNotImplemented + dir, name := genDirAndName(key) + + var data Model + + var where = bson.M{"directory": dir, "name": name} + err = store.connect.Database(store.database).Collection(store.collectionName).FindOne(ctx, where).Decode(&data) + if err != mongo.ErrNoDocuments && err != nil { + glog.Error("kv get: %v", err) + return nil, filer.ErrKvNotFound + } + + if len(data.Meta) == 0 { + return nil, filer.ErrKvNotFound + } + + return data.Meta, nil } func (store *MongodbStore) KvDelete(ctx context.Context, key []byte) (err error) { - return filer.ErrKvNotImplemented + + dir, name := genDirAndName(key) + + where := bson.M{"directory": dir, "name": name} + _, err = store.connect.Database(store.database).Collection(store.collectionName).DeleteOne(ctx, where) + if err != nil { + return fmt.Errorf("kv delete %s : %v", err) + } + + return nil +} + +func genDirAndName(key []byte) (dir string, name string) { + for len(key) < 8 { + key = append(key, 0) + } + + dir = string(key[:8]) + name = string(key[8:]) + + return }