mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge pull request #1431 from kmlebedev/wip-prefix-search
Wip prefix search
This commit is contained in:
commit
edb9d65e05
|
@ -4,7 +4,6 @@ import (
|
|||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
|
@ -164,14 +163,13 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
|
||||
|
||||
func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
sqlText := store.SqlListExclusive
|
||||
if inclusive {
|
||||
sqlText = store.SqlListInclusive
|
||||
}
|
||||
|
||||
rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), limit)
|
||||
rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), prefix, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list %s : %v", fullpath, err)
|
||||
}
|
||||
|
@ -199,6 +197,10 @@ func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpat
|
|||
return entries, nil
|
||||
}
|
||||
|
||||
func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
|
||||
return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
|
||||
}
|
||||
|
||||
func (store *AbstractSqlStore) Shutdown() {
|
||||
store.DB.Close()
|
||||
}
|
||||
|
|
|
@ -3,7 +3,6 @@ package cassandra
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/gocql/gocql"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||
|
@ -126,6 +125,10 @@ func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
return nil, filer2.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool,
|
||||
limit int) (entries []*filer2.Entry, err error) {
|
||||
|
||||
|
|
|
@ -135,9 +135,11 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *EtcdStore) ListDirectoryEntries(
|
||||
ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int,
|
||||
) (entries []*filer2.Entry, err error) {
|
||||
func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
return nil, filer2.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
|
||||
directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
|
||||
|
||||
resp, err := store.client.Get(ctx, string(directoryPrefix),
|
||||
|
|
|
@ -2,6 +2,7 @@ package filer2
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
@ -22,8 +23,9 @@ const (
|
|||
)
|
||||
|
||||
var (
|
||||
OS_UID = uint32(os.Getuid())
|
||||
OS_GID = uint32(os.Getgid())
|
||||
OS_UID = uint32(os.Getuid())
|
||||
OS_GID = uint32(os.Getgid())
|
||||
ErrUnsupportedListDirectoryPrefixed = errors.New("UNSUPPORTED")
|
||||
)
|
||||
|
||||
type Filer struct {
|
||||
|
@ -246,15 +248,15 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
|
|||
|
||||
}
|
||||
|
||||
func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
|
||||
func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) ([]*Entry, error) {
|
||||
if strings.HasSuffix(string(p), "/") && len(p) > 1 {
|
||||
p = p[0 : len(p)-1]
|
||||
}
|
||||
|
||||
var makeupEntries []*Entry
|
||||
entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
|
||||
entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix)
|
||||
for expiredCount > 0 && err == nil {
|
||||
makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount)
|
||||
makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix)
|
||||
if err == nil {
|
||||
entries = append(entries, makeupEntries...)
|
||||
}
|
||||
|
@ -263,8 +265,8 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
|
|||
return entries, err
|
||||
}
|
||||
|
||||
func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int) (entries []*Entry, expiredCount int, lastFileName string, err error) {
|
||||
listedEntries, listErr := f.Store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
|
||||
func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*Entry, expiredCount int, lastFileName string, err error) {
|
||||
listedEntries, listErr := f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix)
|
||||
if listErr != nil {
|
||||
return listedEntries, expiredCount, "", listErr
|
||||
}
|
||||
|
|
|
@ -29,7 +29,7 @@ func (f *Filer) LoadBuckets() {
|
|||
|
||||
limit := math.MaxInt32
|
||||
|
||||
entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit)
|
||||
entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "")
|
||||
|
||||
if err != nil {
|
||||
glog.V(1).Infof("no buckets found: %v", err)
|
||||
|
|
|
@ -58,7 +58,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
|||
lastFileName := ""
|
||||
includeLastFile := false
|
||||
for {
|
||||
entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize)
|
||||
entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "")
|
||||
if err != nil {
|
||||
glog.Errorf("list folder %s: %v", entry.FullPath, err)
|
||||
return nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err)
|
||||
|
|
|
@ -100,13 +100,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
|
|||
sizeBuf := make([]byte, 4)
|
||||
startTsNs := startTime.UnixNano()
|
||||
|
||||
dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366)
|
||||
dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "")
|
||||
if listDayErr != nil {
|
||||
return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
|
||||
}
|
||||
for _, dayEntry := range dayEntries {
|
||||
// println("checking day", dayEntry.FullPath)
|
||||
hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60)
|
||||
hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "")
|
||||
if listHourMinuteErr != nil {
|
||||
return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package filer2
|
|||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
|
@ -21,6 +22,7 @@ type FilerStore interface {
|
|||
DeleteEntry(context.Context, util.FullPath) (err error)
|
||||
DeleteFolderChildren(context.Context, util.FullPath) (err error)
|
||||
ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error)
|
||||
ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error)
|
||||
|
||||
BeginTransaction(ctx context.Context) (context.Context, error)
|
||||
CommitTransaction(ctx context.Context) error
|
||||
|
@ -135,6 +137,54 @@ func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath
|
|||
return entries, err
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int, prefix string) ([]*Entry, error) {
|
||||
stats.FilerStoreCounter.WithLabelValues(fsw.ActualStore.GetName(), "list").Inc()
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
stats.FilerStoreHistogram.WithLabelValues(fsw.ActualStore.GetName(), "list").Observe(time.Since(start).Seconds())
|
||||
}()
|
||||
entries, err := fsw.ActualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix)
|
||||
if err == ErrUnsupportedListDirectoryPrefixed {
|
||||
count := 0
|
||||
notPrefixed, err := fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if prefix == "" {
|
||||
entries = notPrefixed
|
||||
} else {
|
||||
var lastFileName string
|
||||
for count < limit {
|
||||
for _, entry := range notPrefixed {
|
||||
lastFileName = entry.Name()
|
||||
if strings.HasPrefix(entry.Name(), prefix) {
|
||||
count++
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if count >= limit {
|
||||
goto Exit
|
||||
}
|
||||
}
|
||||
notPrefixed, err = fsw.ActualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(notPrefixed) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
Exit:
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
filer_pb.AfterEntryDeserialization(entry.Chunks)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) {
|
||||
return fsw.ActualStore.BeginTransaction(ctx)
|
||||
}
|
||||
|
|
|
@ -4,9 +4,8 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
|
||||
|
||||
|
@ -50,7 +49,7 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
|
|||
}
|
||||
|
||||
if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
|
||||
if errors.IsCorrupted(err) {
|
||||
if leveldb_errors.IsCorrupted(err) {
|
||||
store.db, err = leveldb.RecoverFile(dir, opts)
|
||||
}
|
||||
if err != nil {
|
||||
|
@ -159,6 +158,10 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
return nil, filer2.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
|
||||
limit int) (entries []*filer2.Entry, err error) {
|
||||
|
||||
|
|
|
@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
|
|||
}
|
||||
|
||||
// checking one upper directory
|
||||
entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100)
|
||||
entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "")
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("list entries count: %v", len(entries))
|
||||
return
|
||||
}
|
||||
|
||||
// checking one upper directory
|
||||
entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100)
|
||||
entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "")
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("list entries count: %v", len(entries))
|
||||
return
|
||||
|
@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
|
|||
ctx := context.Background()
|
||||
|
||||
// checking one upper directory
|
||||
entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100)
|
||||
entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "")
|
||||
if err != nil {
|
||||
t.Errorf("list entries: %v", err)
|
||||
return
|
||||
|
|
|
@ -5,13 +5,12 @@ import (
|
|||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/errors"
|
||||
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
@ -53,7 +52,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
|
|||
dbFolder := fmt.Sprintf("%s/%02d", dir, d)
|
||||
os.MkdirAll(dbFolder, 0755)
|
||||
db, dbErr := leveldb.OpenFile(dbFolder, opts)
|
||||
if errors.IsCorrupted(dbErr) {
|
||||
if leveldb_errors.IsCorrupted(dbErr) {
|
||||
db, dbErr = leveldb.RecoverFile(dbFolder, opts)
|
||||
}
|
||||
if dbErr != nil {
|
||||
|
@ -168,6 +167,10 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
return nil, filer2.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
|
||||
limit int) (entries []*filer2.Entry, err error) {
|
||||
|
||||
|
|
|
@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
|
|||
}
|
||||
|
||||
// checking one upper directory
|
||||
entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100)
|
||||
entries, _ := filer.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "")
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("list entries count: %v", len(entries))
|
||||
return
|
||||
}
|
||||
|
||||
// checking one upper directory
|
||||
entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100)
|
||||
entries, _ = filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "")
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("list entries count: %v", len(entries))
|
||||
return
|
||||
|
@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
|
|||
ctx := context.Background()
|
||||
|
||||
// checking one upper directory
|
||||
entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100)
|
||||
entries, err := filer.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "")
|
||||
if err != nil {
|
||||
t.Errorf("list entries: %v", err)
|
||||
return
|
||||
|
|
|
@ -167,6 +167,10 @@ func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath ut
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
return nil, filer2.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
|
||||
|
||||
var where = bson.M{"directory": string(fullpath), "name": bson.M{"$gt": startFileName}}
|
||||
|
|
|
@ -41,14 +41,14 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
|
|||
|
||||
func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int,
|
||||
interpolateParams bool) (err error) {
|
||||
|
||||
//
|
||||
store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)"
|
||||
store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?"
|
||||
store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?"
|
||||
store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?"
|
||||
store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=? AND directory=?"
|
||||
store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? ORDER BY NAME ASC LIMIT ?"
|
||||
store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? ORDER BY NAME ASC LIMIT ?"
|
||||
store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? AND name like CONCAT(?,'%') ORDER BY NAME ASC LIMIT ?"
|
||||
store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? AND name like CONCAT(?,'%') ORDER BY NAME ASC LIMIT ?"
|
||||
|
||||
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
|
||||
if interpolateParams {
|
||||
|
|
|
@ -46,8 +46,8 @@ func (store *PostgresStore) initialize(user, password, hostname string, port int
|
|||
store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3"
|
||||
store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3"
|
||||
store.SqlDeleteFolderChildren = "DELETE FROM filemeta WHERE dirhash=$1 AND directory=$2"
|
||||
store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4"
|
||||
store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4"
|
||||
store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like CONCAT($4,'%')ORDER BY NAME ASC LIMIT $5"
|
||||
store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like CONCAT($4,'%') ORDER BY NAME ASC LIMIT $5"
|
||||
|
||||
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, user, sslmode)
|
||||
if password != "" {
|
||||
|
|
|
@ -121,6 +121,10 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedisStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
return nil, filer2.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool,
|
||||
limit int) (entries []*filer2.Entry, err error) {
|
||||
|
||||
|
|
|
@ -116,6 +116,10 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful
|
|||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer2.Entry, err error) {
|
||||
return nil, filer2.ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool,
|
||||
limit int) (entries []*filer2.Entry, err error) {
|
||||
|
||||
|
|
|
@ -6,7 +6,6 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer2"
|
||||
|
@ -59,7 +58,7 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file
|
|||
lastFileName := req.StartFromFileName
|
||||
includeLastFile := req.InclusiveStartFrom
|
||||
for limit > 0 {
|
||||
entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit)
|
||||
entries, err := fs.filer.ListDirectoryEntries(stream.Context(), util.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit, req.Prefix)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -74,12 +73,6 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file
|
|||
|
||||
lastFileName = entry.Name()
|
||||
|
||||
if req.Prefix != "" {
|
||||
if !strings.HasPrefix(entry.Name(), req.Prefix) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := stream.Send(&filer_pb.ListEntriesResponse{
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: entry.Name(),
|
||||
|
|
|
@ -70,7 +70,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
|
|||
includeLastFile := false
|
||||
for {
|
||||
|
||||
entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024)
|
||||
entries, err := fs.filer.ListDirectoryEntries(ctx, currentDirPath, lastFileName, includeLastFile, 1024, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -32,7 +32,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
|
||||
lastFileName := r.FormValue("lastFileName")
|
||||
|
||||
entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit)
|
||||
entries, err := fs.filer.ListDirectoryEntries(context.Background(), util.FullPath(path), lastFileName, false, limit, "")
|
||||
|
||||
if err != nil {
|
||||
glog.V(0).Infof("listDirectory %s %s %d: %s", path, lastFileName, limit, err)
|
||||
|
|
Loading…
Reference in a new issue