filer: increase directory listing pagination size

This commit is contained in:
Chris Lu 2019-12-12 09:11:10 -08:00
parent bf4b13612d
commit d0b423bbc0
5 changed files with 15 additions and 14 deletions

View file

@ -3,18 +3,22 @@ package filer2
import (
"context"
"fmt"
"google.golang.org/grpc"
"math"
"os"
"path/filepath"
"strings"
"time"
"google.golang.org/grpc"
"github.com/karlseguin/ccache"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/karlseguin/ccache"
)
const PaginationSize = 1024 * 256
var (
OS_UID = uint32(os.Getuid())
OS_GID = uint32(os.Getgid())
@ -32,7 +36,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
f := &Filer{
directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
MasterClient: wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
fileIdDeletionChan: make(chan string, 4096),
fileIdDeletionChan: make(chan string, PaginationSize),
GrpcDialOption: grpcDialOption,
}
@ -217,7 +221,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
lastFileName := ""
includeLastFile := false
for limit > 0 {
entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, PaginationSize)
if err != nil {
glog.Errorf("list folder %s: %v", p, err)
return fmt.Errorf("list folder %s: %v", p, err)
@ -241,7 +245,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
}
}
if len(entries) < 1024 {
if len(entries) < PaginationSize {
break
}
}

View file

@ -128,8 +128,6 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
paginationLimit := 1024 * 256
lastEntryName := ""
for {
@ -137,7 +135,7 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
request := &filer_pb.ListEntriesRequest{
Directory: fullDirPath,
StartFromFileName: lastEntryName,
Limit: uint32(paginationLimit),
Limit: PaginationSize,
}
glog.V(3).Infof("read directory: %v", request)
@ -151,7 +149,7 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
lastEntryName = entry.Name
}
if len(resp.Entries) < paginationLimit {
if len(resp.Entries) < PaginationSize {
break
}

View file

@ -214,7 +214,6 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
paginationLimit := 1024 * 256
remaining := dir.wfs.option.DirListingLimit
lastEntryName := ""
@ -224,7 +223,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
request := &filer_pb.ListEntriesRequest{
Directory: dir.Path,
StartFromFileName: lastEntryName,
Limit: uint32(paginationLimit),
Limit: filer2.PaginationSize,
}
glog.V(4).Infof("read directory: %v", request)
@ -250,7 +249,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
remaining -= len(resp.Entries)
if len(resp.Entries) < paginationLimit {
if len(resp.Entries) < filer2.PaginationSize {
break
}

View file

@ -107,7 +107,7 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str
ret = append(ret, result...)
}
glog.V(0).Infof("deleted %d items", len(ret))
glog.V(1).Infof("deleted %d items", len(ret))
return ret, err
}

View file

@ -40,7 +40,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
limit = fs.option.DirListingLimit
}
paginationLimit := 1024 * 256
paginationLimit := filer2.PaginationSize
if limit < paginationLimit {
paginationLimit = limit
}