From 33a9e5e2d1948d08ec6456be6df3a6683a780905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 22:19:16 +0500 Subject: [PATCH 01/15] test ListDirectoryPrefixedEntries --- .../filer2/abstract_sql/abstract_sql_store.go | 69 ++++++++++++++++++- weed/filer2/cassandra/cassandra_store.go | 31 +++++++++ weed/filer2/etcd/etcd_store.go | 34 ++++++++- weed/filer2/filer.go | 10 +-- weed/filer2/filerstore.go | 5 +- weed/filer2/leveldb/leveldb_store.go | 31 +++++++++ weed/filer2/leveldb2/leveldb2_store.go | 31 +++++++++ weed/filer2/mongodb/mongodb_store.go | 31 +++++++++ weed/filer2/mysql/mysql_store.go | 2 +- weed/filer2/redis/universal_redis_store.go | 30 ++++++++ weed/filer2/redis2/universal_redis_store.go | 31 +++++++++ weed/server/filer_grpc_server.go | 9 +-- 12 files changed, 294 insertions(+), 20 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index 5ade18960..ed0d6e8ef 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -150,8 +150,75 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { +//func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +// sqlText := store.SqlListExclusive +// if inclusive { +// sqlText = store.SqlListInclusive +// } +// +// rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit) +// if err != nil { +// return nil, fmt.Errorf("list %s : %v", fullpath, err) +// } +// defer rows.Close() +// +// for rows.Next() { +// var name string +// var data []byte +// if err = rows.Scan(&name, &data); err != nil { +// glog.V(0).Infof("scan %s : %v", fullpath, err) +// return nil, fmt.Errorf("scan %s: %v", fullpath, err) +// } +// +// entry := &filer2.Entry{ +// FullPath: util.NewFullPath(string(fullpath), name), +// } +// if err = entry.DecodeAttributesAndChunks(data); err != nil { +// glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) +// return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) +// } +// +// entries = append(entries, entry) +// } +// +// return entries, nil +//} +//func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +// return nil, fmt.Errorf("not implemented") +// +//} +func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { sqlText := store.SqlListExclusive if inclusive { sqlText = store.SqlListInclusive diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 5dd7d8036..225ad02a3 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -3,6 +3,7 @@ package cassandra import ( "context" "fmt" + "strings" "github.com/gocql/gocql" @@ -126,6 +127,36 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath return nil } +func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 2ef65b4a0..5fbdb60aa 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -135,9 +135,37 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryEntries( - ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, -) (entries []*filer2.Entry, err error) { +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + +func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { directoryPrefix := genDirectoryKeyPrefix(fullpath, "") resp, err := store.client.Get(ctx, string(directoryPrefix), diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index dd4c38857..92764f600 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -259,15 +259,15 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } -func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) { +func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) ([]*Entry, error) { if strings.HasSuffix(string(p), "/") && len(p) > 1 { p = p[0 : len(p)-1] } var makeupEntries []*Entry - entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit) + entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) for expiredCount > 0 && err == nil { - makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount) + makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix) if err == nil { entries = append(entries, makeupEntries...) } @@ -276,8 +276,8 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start return entries, err } -func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int) (entries []*Entry, expiredCount int, lastFileName string, err error) { - listedEntries, listErr := f.Store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit) +func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) { + listedEntries, listErr := f.Store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) if listErr != nil { return listedEntries, expiredCount, "", listErr } diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 7c518c6fe..3df43e6b2 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -21,6 +21,7 @@ type FilerStore interface { DeleteEntry(context.Context, util.FullPath) (err error) DeleteFolderChildren(context.Context, util.FullPath) (err error) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) + ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error @@ -112,14 +113,14 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. return fsw.ActualStore.DeleteFolderChildren(ctx, fp) } -func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) { +func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc() start := time.Now() defer func() { stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() - entries, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) if err != nil { return nil, err } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 31919ca49..4b42340e0 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "strings" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" @@ -159,6 +160,36 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index c907e8746..46b2558cb 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "strings" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" @@ -168,6 +169,36 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 375a457a4..a1502430c 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -11,6 +11,7 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/x/bsonx" + "strings" "time" ) @@ -167,6 +168,36 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut return nil } +func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { var where = bson.M{"directory": string(fullpath), "name": bson.M{"$gt": startFileName}} diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index 63d99cd9d..b22ac19fb 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -41,7 +41,7 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int, interpolateParams bool) (err error) { - + //AND name like CONCAT(?,'%') store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index e5b9e8840..0c90b8993 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -121,6 +121,36 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full return nil } +func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 420336b46..483dd621f 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -3,6 +3,7 @@ package redis2 import ( "context" "fmt" + "strings" "time" "github.com/go-redis/redis" @@ -116,6 +117,36 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful return nil } +func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + count := 0 + notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + for count < limit { + for _, entry := range notPrefixed { + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + if err != nil { + return nil, err + } + } + + return entries, nil +} + func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 48e9253f0..eaace2fc2 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "strconv" - "strings" "time" "github.com/chrislusf/seaweedfs/weed/filer2" @@ -59,7 +58,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file lastFileName := req.StartFromFileName includeLastFile := req.InclusiveStartFrom for limit > 0 { - entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit) + entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit, req.Prefix) if err != nil { return err @@ -74,12 +73,6 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file lastFileName = entry.Name() - if req.Prefix != "" { - if !strings.HasPrefix(entry.Name(), req.Prefix) { - continue - } - } - if err := stream.Send(&filer_pb.ListEntriesResponse{ Entry: &filer_pb.Entry{ Name: entry.Name(), From dc9fc0125419028da895c01f754305891fce2eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 23:38:00 +0500 Subject: [PATCH 02/15] test ListDirectoryPrefixedEntries --- .../filer2/abstract_sql/abstract_sql_store.go | 1 + weed/filer2/etcd/etcd_store.go | 2 +- weed/filer2/filer.go | 2 +- weed/filer2/filer_buckets.go | 2 +- weed/filer2/filer_delete_entry.go | 2 +- weed/filer2/filer_notify.go | 4 ++-- weed/filer2/filerstore.go | 21 +++++++++++++++++-- weed/filer2/leveldb/leveldb_store.go | 2 +- weed/filer2/leveldb2/leveldb2_store.go | 2 +- 9 files changed, 28 insertions(+), 10 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index ed0d6e8ef..b6da8e74e 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 5fbdb60aa..a1bf9bf3c 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -135,7 +135,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ return nil } -func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { count := 0 notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) if err != nil { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 92764f600..cc8fecc42 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -277,7 +277,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start } func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) { - listedEntries, listErr := f.Store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix) + listedEntries, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix) if listErr != nil { return listedEntries, expiredCount, "", listErr } diff --git a/weed/filer2/filer_buckets.go b/weed/filer2/filer_buckets.go index 7a57e7ee1..6b7c2c31a 100644 --- a/weed/filer2/filer_buckets.go +++ b/weed/filer2/filer_buckets.go @@ -29,7 +29,7 @@ func (f *Filer) LoadBuckets() { limit := math.MaxInt32 - entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit) + entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "") if err != nil { glog.V(1).Infof("no buckets found: %v", err) diff --git a/weed/filer2/filer_delete_entry.go b/weed/filer2/filer_delete_entry.go index 35099a472..17624bbd6 100644 --- a/weed/filer2/filer_delete_entry.go +++ b/weed/filer2/filer_delete_entry.go @@ -58,7 +58,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry lastFileName := "" includeLastFile := false for { - entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize) + entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "") if err != nil { glog.Errorf("list folder %s: %v", entry.FullPath, err) return nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go index e5f9eba0a..97ae8e9c8 100644 --- a/weed/filer2/filer_notify.go +++ b/weed/filer2/filer_notify.go @@ -96,13 +96,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func( sizeBuf := make([]byte, 4) startTsNs := startTime.UnixNano() - dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366) + dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "") if listDayErr != nil { return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr) } for _, dayEntry := range dayEntries { // println("checking day", dayEntry.FullPath) - hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60) + hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "") if listHourMinuteErr != nil { return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) } diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 3df43e6b2..8edbe3034 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -113,7 +113,24 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util. return fsw.ActualStore.DeleteFolderChildren(ctx, fp) } -func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { +func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) { + stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc() + start := time.Now() + defer func() { + stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) + }() + + entries, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + for _, entry := range entries { + filer_pb.AfterEntryDeserialization(entry.Chunks) + } + return entries, err +} + +func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) { stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc() start := time.Now() defer func() { @@ -127,7 +144,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath for _, entry := range entries { filer_pb.AfterEntryDeserialization(entry.Chunks) } - return entries, err + return entries, nil } func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) { diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 4b42340e0..5b118884e 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -160,7 +160,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we return nil } -func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { count := 0 notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) if err != nil { diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 46b2558cb..fdff3c92d 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -169,7 +169,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w return nil } -func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { +func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { count := 0 notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) if err != nil { From 67f134ebd7c0a84e06f36979886afdd7e3435eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 23:44:48 +0500 Subject: [PATCH 03/15] test ListDirectoryPrefixedEntries --- weed/filer2/leveldb/leveldb_store_test.go | 6 +++--- weed/filer2/leveldb2/leveldb2_store_test.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/weed/filer2/leveldb/leveldb_store_test.go b/weed/filer2/leveldb/leveldb_store_test.go index 77df07a9b..09dde72d4 100644 --- a/weed/filer2/leveldb/leveldb_store_test.go +++ b/weed/filer2/leveldb/leveldb_store_test.go @@ -50,14 +50,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100) + entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if err != nil { t.Errorf("list entries: %v", err) return diff --git a/weed/filer2/leveldb2/leveldb2_store_test.go b/weed/filer2/leveldb2/leveldb2_store_test.go index b211d86e4..d212d24ba 100644 --- a/weed/filer2/leveldb2/leveldb2_store_test.go +++ b/weed/filer2/leveldb2/leveldb2_store_test.go @@ -50,14 +50,14 @@ func TestCreateAndFind(t *testing.T) { } // checking one upper directory - entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100) + entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return } // checking one upper directory - entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) { ctx := context.Background() // checking one upper directory - entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100) + entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "") if err != nil { t.Errorf("list entries: %v", err) return From 2d3b355fb6a5060ec32583478598a13344c90f74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 5 Aug 2020 23:56:06 +0500 Subject: [PATCH 04/15] test ListDirectoryPrefixedEntries --- weed/server/filer_grpc_server_rename.go | 2 +- weed/server/filer_server_handlers_read_dir.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index 9642fec24..219067c86 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -70,7 +70,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util. includeLastFile := false for { - entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024) + entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "") if err != nil { return err } diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index ae28fc1db..9ca0209f4 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -32,7 +32,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque lastFileName := r.FormValue("lastFileName") - entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit) + entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, "") if err != nil { glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err) From a457c308ad31cfa90919fb9f430aeb9faa3b06d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 6 Aug 2020 00:24:31 +0500 Subject: [PATCH 05/15] test ListDirectoryPrefixedEntries --- weed/filer2/abstract_sql/abstract_sql_store.go | 8 +++++++- weed/filer2/cassandra/cassandra_store.go | 8 +++++++- weed/filer2/etcd/etcd_store.go | 8 +++++++- weed/filer2/leveldb/leveldb_store.go | 8 +++++++- weed/filer2/leveldb2/leveldb2_store.go | 8 +++++++- weed/filer2/mongodb/mongodb_store.go | 8 +++++++- weed/filer2/redis/universal_redis_store.go | 8 +++++++- weed/filer2/redis2/universal_redis_store.go | 8 +++++++- 8 files changed, 56 insertions(+), 8 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index b6da8e74e..a90c1341f 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -199,8 +199,10 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -210,10 +212,14 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 225ad02a3..dc29371d8 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -137,8 +137,10 @@ func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, f if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -148,10 +150,14 @@ func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, f break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index a1bf9bf3c..9b4e21917 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -145,8 +145,10 @@ func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpa if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -156,10 +158,14 @@ func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpa break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 5b118884e..0d6a62212 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -170,8 +170,10 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -181,10 +183,14 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, ful break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index fdff3c92d..a8e915e79 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -179,8 +179,10 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -190,10 +192,14 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fu break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index a1502430c..00f710f1f 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -178,8 +178,10 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, ful if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -189,10 +191,14 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, ful break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 0c90b8993..a3a8f866e 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -131,8 +131,10 @@ func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Conte if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -142,10 +144,14 @@ func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Conte break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 483dd621f..289e85545 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -127,8 +127,10 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont if prefix == "" { return notPrefixed, nil } + var lastFileName string for count < limit { for _, entry := range notPrefixed { + lastFileName = entry.Name() if strings.HasPrefix(entry.Name(), prefix) { count++ entries = append(entries, entry) @@ -138,10 +140,14 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont break } - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) + notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) if err != nil { return nil, err } + + if len(notPrefixed) == 0 { + break + } } return entries, nil From b231f7bdab084683099f429631e4d1dcfb6ae7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 6 Aug 2020 00:37:42 +0500 Subject: [PATCH 06/15] ListDirectoryPrefixedEntries --- .../filer2/abstract_sql/abstract_sql_store.go | 82 ++----------------- weed/filer2/mysql/mysql_store.go | 6 +- weed/filer2/postgres/postgres_store.go | 4 +- 3 files changed, 10 insertions(+), 82 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index a90c1341f..fb0cadee5 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -4,8 +4,6 @@ import ( "context" "database/sql" "fmt" - "strings" - "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -151,87 +149,13 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return nil } -//func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { -// sqlText := store.SqlListExclusive -// if inclusive { -// sqlText = store.SqlListInclusive -// } -// -// rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit) -// if err != nil { -// return nil, fmt.Errorf("list %s : %v", fullpath, err) -// } -// defer rows.Close() -// -// for rows.Next() { -// var name string -// var data []byte -// if err = rows.Scan(&name, &data); err != nil { -// glog.V(0).Infof("scan %s : %v", fullpath, err) -// return nil, fmt.Errorf("scan %s: %v", fullpath, err) -// } -// -// entry := &filer2.Entry{ -// FullPath: util.NewFullPath(string(fullpath), name), -// } -// if err = entry.DecodeAttributesAndChunks(data); err != nil { -// glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) -// return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) -// } -// -// entries = append(entries, entry) -// } -// -// return entries, nil -//} -//func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { -// return nil, fmt.Errorf("not implemented") -// -//} - func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil -} - -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { sqlText := store.SqlListExclusive if inclusive { sqlText = store.SqlListInclusive } - rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), limit) + rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit) if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -258,6 +182,10 @@ func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpat return entries, nil } +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { + return nil, fmt.Errorf("not implemented") + +} func (store *AbstractSqlStore) Shutdown() { store.DB.Close() diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index b22ac19fb..9f748445e 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -41,14 +41,14 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int, interpolateParams bool) (err error) { - //AND name like CONCAT(?,'%') + // store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?" store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=? AND directory=?" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? ORDER BY NAME ASC LIMIT ?" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? ORDER BY NAME ASC LIMIT ?" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like CONCAT(?,'%') ORDER BY NAME ASC LIMIT ?" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like CONCAT(?,'%') ORDER BY NAME ASC LIMIT ?" sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) if interpolateParams { diff --git a/weed/filer2/postgres/postgres_store.go b/weed/filer2/postgres/postgres_store.go index 51c069aae..87eb6aca2 100644 --- a/weed/filer2/postgres/postgres_store.go +++ b/weed/filer2/postgres/postgres_store.go @@ -46,8 +46,8 @@ func (store *PostgresStore) initialize(user, password, hostname string, port int store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=$1 AND directory=$2" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like CONCAT($4,'%')ORDER BY NAME ASC LIMIT $5" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like CONCAT($4,'%') ORDER BY NAME ASC LIMIT $5" sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, user, sslmode) if password != "" { From 9ca011e3cab51f7f0489df463ba4566216d8cf8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A3=D1=81=D1=82=D1=8E=D0=B6=D0=B0=D0=BD=D0=B8=D0=BD=20?= =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 6 Aug 2020 00:38:59 +0500 Subject: [PATCH 07/15] ListDirectoryPrefixedEntries --- weed/filer2/abstract_sql/abstract_sql_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index fb0cadee5..d1ce83de6 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -182,8 +182,8 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } -func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("not implemented") +func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { + return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") } From 2ea638f865c9bebe89c05d8a4acf90a1447375f3 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 17:13:56 +0500 Subject: [PATCH 08/15] filer store wrapper can implement the logic to filter by prefi --- .../filer2/abstract_sql/abstract_sql_store.go | 2 +- weed/filer2/cassandra/cassandra_store.go | 36 +-------------- weed/filer2/etcd/etcd_store.go | 34 +------------- weed/filer2/filerstore.go | 44 ++++++++++++++++++- weed/filer2/leveldb/leveldb_store.go | 36 +-------------- weed/filer2/leveldb2/leveldb2_store.go | 40 ++--------------- weed/filer2/mongodb/mongodb_store.go | 35 +-------------- weed/filer2/redis/universal_redis_store.go | 34 +------------- weed/filer2/redis2/universal_redis_store.go | 35 +-------------- 9 files changed, 53 insertions(+), 243 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index d1ce83de6..7f591413a 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -182,9 +182,9 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } + func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "") - } func (store *AbstractSqlStore) Shutdown() { diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index dc29371d8..a996a6401 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -3,8 +3,6 @@ package cassandra import ( "context" "fmt" - "strings" - "github.com/gocql/gocql" "github.com/chrislusf/seaweedfs/weed/filer2" @@ -128,39 +126,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 9b4e21917..043bf995d 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -136,39 +136,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 8edbe3034..8d4e75e9a 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -2,6 +2,8 @@ package filer2 import ( "context" + "fmt" + "strings" "time" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -22,6 +24,7 @@ type FilerStore interface { DeleteFolderChildren(context.Context, util.FullPath) (err error) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) + ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error @@ -136,8 +139,10 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, defer func() { stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() - entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + if err == fmt.Errorf("UNSUPPORTED") { + entries, err = fsw.ListDirectoryUnSupPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) + } if err != nil { return nil, err } @@ -147,6 +152,43 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } +func (fsw *FilerStoreWrapper) ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { + count := 0 + notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + + if prefix == "" { + return notPrefixed, nil + } + + var lastFileName string + for count < limit { + for _, entry := range notPrefixed { + lastFileName = entry.Name() + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + + if len(notPrefixed) == 0 { + break + } + } + + return entries, nil +} + func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) { return fsw.ActualStore.BeginTransaction(ctx) } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 0d6a62212..104e3d026 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -4,8 +4,6 @@ import ( "bytes" "context" "fmt" - "strings" - "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" @@ -161,39 +159,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index a8e915e79..478661d0e 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -5,14 +5,12 @@ import ( "context" "crypto/md5" "fmt" - "io" - "os" - "strings" - "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + "io" + "os" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" @@ -170,39 +168,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 00f710f1f..1fa18f872 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -11,7 +11,6 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/x/bsonx" - "strings" "time" ) @@ -169,39 +168,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index a3a8f866e..9c71b045d 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -122,39 +122,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 289e85545..6490f7bed 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -3,7 +3,6 @@ package redis2 import ( "context" "fmt" - "strings" "time" "github.com/go-redis/redis" @@ -118,39 +117,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - count := 0 - notPrefixed, err := store.ListDirectoryEntries(ctx, fullpath, startFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = store.ListDirectoryEntries(ctx, fullpath, lastFileName, inclusive, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } - - return entries, nil + return nil, fmt.Errorf("UNSUPPORTED") } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From 82ea121d091e2b498b48e19ee8f7f5e7f48f15e2 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 18:50:06 +0500 Subject: [PATCH 09/15] rm func --- weed/filer2/filerstore.go | 71 ++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 8d4e75e9a..9ea39cf11 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -2,7 +2,6 @@ package filer2 import ( "context" - "fmt" "strings" "time" @@ -24,7 +23,6 @@ type FilerStore interface { DeleteFolderChildren(context.Context, util.FullPath) (err error) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) - ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) BeginTransaction(ctx context.Context) (context.Context, error) CommitTransaction(ctx context.Context) error @@ -140,10 +138,39 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err == fmt.Errorf("UNSUPPORTED") { - entries, err = fsw.ListDirectoryUnSupPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - } - if err != nil { + if err.Error() == "UNSUPPORTED" { + count := 0 + notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + if prefix == "" { + entries = notPrefixed + } else { + var lastFileName string + for count < limit { + for _, entry := range notPrefixed { + lastFileName = entry.Name() + if strings.HasPrefix(entry.Name(), prefix) { + count++ + entries = append(entries, entry) + } + } + if count >= limit { + break + } + + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) + if err != nil { + return nil, err + } + + if len(notPrefixed) == 0 { + break + } + } + } + } else if err != nil { return nil, err } for _, entry := range entries { @@ -153,38 +180,6 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, } func (fsw *FilerStoreWrapper) ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { - count := 0 - notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) - if err != nil { - return nil, err - } - - if prefix == "" { - return notPrefixed, nil - } - - var lastFileName string - for count < limit { - for _, entry := range notPrefixed { - lastFileName = entry.Name() - if strings.HasPrefix(entry.Name(), prefix) { - count++ - entries = append(entries, entry) - } - } - if count >= limit { - break - } - - notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) - if err != nil { - return nil, err - } - - if len(notPrefixed) == 0 { - break - } - } return entries, nil } From 60a86cfe0496497ae4942d3e8925159154faab54 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:35:16 +0500 Subject: [PATCH 10/15] add const --- weed/filer2/cassandra/cassandra_store.go | 3 ++- weed/filer2/etcd/etcd_store.go | 3 ++- weed/filer2/filer.go | 5 ++++- weed/filer2/filerstore.go | 7 +------ weed/filer2/leveldb/leveldb_store.go | 7 ++++--- weed/filer2/leveldb2/leveldb2_store.go | 7 ++++--- weed/filer2/mongodb/mongodb_store.go | 3 ++- weed/filer2/redis/universal_redis_store.go | 3 ++- weed/filer2/redis2/universal_redis_store.go | 3 ++- 9 files changed, 23 insertions(+), 18 deletions(-) diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index a996a6401..8214935f9 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -2,6 +2,7 @@ package cassandra import ( "context" + "errors" "fmt" "github.com/gocql/gocql" @@ -126,7 +127,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 043bf995d..93f3f3781 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -2,6 +2,7 @@ package etcd import ( "context" + "errors" "fmt" "strings" "time" @@ -136,7 +137,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index cc8fecc42..ef81cae2b 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -18,7 +18,10 @@ import ( "github.com/chrislusf/seaweedfs/weed/wdclient" ) -const PaginationSize = 1024 * 256 +const ( + PaginationSize = 1024 * 256 + UnsupportedListDirectoryPrefixedErr = "UNSUPPORTED" +) var ( OS_UID = uint32(os.Getuid()) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 9ea39cf11..3c1e555b1 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == "UNSUPPORTED" { + if err.Error() == UnsupportedListDirectoryPrefixedErr { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { @@ -179,11 +179,6 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, return entries, nil } -func (fsw *FilerStoreWrapper) ListDirectoryUnSupPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) (entries []*Entry, err error) { - - return entries, nil -} - func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) { return fsw.ActualStore.BeginTransaction(ctx) } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 104e3d026..787671c55 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -3,9 +3,10 @@ package leveldb import ( "bytes" "context" + "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" + leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" @@ -49,7 +50,7 @@ func (store *LevelDBStore) initialize(dir string) (err error) { } if store.db, err = leveldb.OpenFile(dir, opts); err != nil { - if errors.IsCorrupted(err) { + if leveldb_errors.IsCorrupted(err) { store.db, err = leveldb.RecoverFile(dir, opts) } if err != nil { @@ -159,7 +160,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 478661d0e..4fc110d69 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -4,9 +4,10 @@ import ( "bytes" "context" "crypto/md5" + "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" + leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/opt" leveldb_util "github.com/syndtr/goleveldb/leveldb/util" "io" @@ -52,7 +53,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { dbFolder := fmt.Sprintf("%s/%02d", dir, d) os.MkdirAll(dbFolder, 0755) db, dbErr := leveldb.OpenFile(dbFolder, opts) - if errors.IsCorrupted(dbErr) { + if leveldb_errors.IsCorrupted(dbErr) { db, dbErr = leveldb.RecoverFile(dbFolder, opts) } if dbErr != nil { @@ -168,7 +169,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 1fa18f872..314f36ce2 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -2,6 +2,7 @@ package mongodb import ( "context" + "errors" "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" @@ -168,7 +169,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 9c71b045d..66450b5a7 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -2,6 +2,7 @@ package redis import ( "context" + "errors" "fmt" "sort" "strings" @@ -122,7 +123,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 6490f7bed..9878799cf 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -2,6 +2,7 @@ package redis2 import ( "context" + "errors" "fmt" "time" @@ -117,7 +118,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, fmt.Errorf("UNSUPPORTED") + return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From 7f013d77d6f42b5116d59703520f2e87a6ae2540 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:43:32 +0500 Subject: [PATCH 11/15] add var UnsupportedListDirectoryPrefixedErr --- weed/filer2/cassandra/cassandra_store.go | 3 +-- weed/filer2/etcd/etcd_store.go | 3 +-- weed/filer2/filer.go | 9 +++++---- weed/filer2/filerstore.go | 2 +- weed/filer2/leveldb/leveldb_store.go | 3 +-- weed/filer2/leveldb2/leveldb2_store.go | 3 +-- weed/filer2/mongodb/mongodb_store.go | 3 +-- weed/filer2/redis/universal_redis_store.go | 3 +-- weed/filer2/redis2/universal_redis_store.go | 3 +-- 9 files changed, 13 insertions(+), 19 deletions(-) diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 8214935f9..40f371e2f 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -2,7 +2,6 @@ package cassandra import ( "context" - "errors" "fmt" "github.com/gocql/gocql" @@ -127,7 +126,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index 93f3f3781..d0b1ff202 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -2,7 +2,6 @@ package etcd import ( "context" - "errors" "fmt" "strings" "time" @@ -137,7 +136,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index ef81cae2b..a94052996 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -2,6 +2,7 @@ package filer2 import ( "context" + "errors" "fmt" "os" "strings" @@ -19,13 +20,13 @@ import ( ) const ( - PaginationSize = 1024 * 256 - UnsupportedListDirectoryPrefixedErr = "UNSUPPORTED" + PaginationSize = 1024 * 256 ) var ( - OS_UID = uint32(os.Getuid()) - OS_GID = uint32(os.Getgid()) + OS_UID = uint32(os.Getuid()) + OS_GID = uint32(os.Getgid()) + UnsupportedListDirectoryPrefixedErr = errors.New("UNSUPPORTED") ) type Filer struct { diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 3c1e555b1..642ebb3c8 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == UnsupportedListDirectoryPrefixedErr { + if err.Error() == UnsupportedListDirectoryPrefixedErr.Error() { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 787671c55..2ea88635b 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -3,7 +3,6 @@ package leveldb import ( "bytes" "context" - "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" @@ -160,7 +159,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 4fc110d69..2a9c2fe3f 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/md5" - "errors" "fmt" "github.com/syndtr/goleveldb/leveldb" leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" @@ -169,7 +168,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 314f36ce2..080d6248f 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -2,7 +2,6 @@ package mongodb import ( "context" - "errors" "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" @@ -169,7 +168,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 66450b5a7..8cc70c72b 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -2,7 +2,6 @@ package redis import ( "context" - "errors" "fmt" "sort" "strings" @@ -123,7 +122,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 9878799cf..2ce7ebec7 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -2,7 +2,6 @@ package redis2 import ( "context" - "errors" "fmt" "time" @@ -118,7 +117,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, errors.New(filer2.UnsupportedListDirectoryPrefixedErr) + return nil, filer2.UnsupportedListDirectoryPrefixedErr } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From a8de332beb933db2acc1c93b7c352fb7f2c20647 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:45:44 +0500 Subject: [PATCH 12/15] fix --- weed/filer2/filer.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index a94052996..603e51fa2 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -19,9 +19,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/wdclient" ) -const ( - PaginationSize = 1024 * 256 -) +const PaginationSize = 1024 * 256 var ( OS_UID = uint32(os.Getuid()) From 22f32e75c411fbf13c209ebaeadb673d72a9a29b Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:52:05 +0500 Subject: [PATCH 13/15] rename --- weed/filer2/cassandra/cassandra_store.go | 2 +- weed/filer2/etcd/etcd_store.go | 2 +- weed/filer2/filer.go | 2 +- weed/filer2/filerstore.go | 2 +- weed/filer2/leveldb/leveldb_store.go | 2 +- weed/filer2/leveldb2/leveldb2_store.go | 2 +- weed/filer2/mongodb/mongodb_store.go | 2 +- weed/filer2/redis/universal_redis_store.go | 2 +- weed/filer2/redis2/universal_redis_store.go | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 40f371e2f..4d845c2fa 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -126,7 +126,7 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath } func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/etcd/etcd_store.go b/weed/filer2/etcd/etcd_store.go index d0b1ff202..6f4c3ce5c 100644 --- a/weed/filer2/etcd/etcd_store.go +++ b/weed/filer2/etcd/etcd_store.go @@ -136,7 +136,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_ } func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 603e51fa2..9d485ad8a 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -24,7 +24,7 @@ const PaginationSize = 1024 * 256 var ( OS_UID = uint32(os.Getuid()) OS_GID = uint32(os.Getgid()) - UnsupportedListDirectoryPrefixedErr = errors.New("UNSUPPORTED") + ErrUnsupportedListDirectoryPrefixed = errors.New("UNSUPPORTED") ) type Filer struct { diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index 642ebb3c8..b55aeb4d6 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == UnsupportedListDirectoryPrefixedErr.Error() { + if err.Error() == ErrUnsupportedListDirectoryPrefixed.Error() { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 2ea88635b..1c08d2831 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -159,7 +159,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we } func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go index 2a9c2fe3f..ca9d6f04d 100644 --- a/weed/filer2/leveldb2/leveldb2_store.go +++ b/weed/filer2/leveldb2/leveldb2_store.go @@ -168,7 +168,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w } func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/mongodb/mongodb_store.go b/weed/filer2/mongodb/mongodb_store.go index 080d6248f..661aa4ea0 100644 --- a/weed/filer2/mongodb/mongodb_store.go +++ b/weed/filer2/mongodb/mongodb_store.go @@ -168,7 +168,7 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut } func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 8cc70c72b..fc2abef6c 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -122,7 +122,7 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full } func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go index 2ce7ebec7..c639635ef 100644 --- a/weed/filer2/redis2/universal_redis_store.go +++ b/weed/filer2/redis2/universal_redis_store.go @@ -117,7 +117,7 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful } func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) { - return nil, filer2.UnsupportedListDirectoryPrefixedErr + return nil, filer2.ErrUnsupportedListDirectoryPrefixed } func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, From 18b98f9747082c9cd5e7ee83f98ab3d6c0eeddb3 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 21:55:18 +0500 Subject: [PATCH 14/15] not convert err to string --- weed/filer2/filerstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index b55aeb4d6..db53734bc 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -138,7 +138,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds()) }() entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix) - if err.Error() == ErrUnsupportedListDirectoryPrefixed.Error() { + if err == ErrUnsupportedListDirectoryPrefixed { count := 0 notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit) if err != nil { From 9a195bebfd6c803161d07ca80b227dd058719aa5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 31 Aug 2020 22:13:13 +0500 Subject: [PATCH 15/15] accurate limit --- weed/filer2/filerstore.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index db53734bc..32c4c94fb 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -155,12 +155,11 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, count++ entries = append(entries, entry) } + if count >= limit { + goto Exit + } } - if count >= limit { - break - } - - notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, includeStartFile, limit) + notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit) if err != nil { return nil, err } @@ -169,6 +168,7 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, break } } + Exit: } } else if err != nil { return nil, err