filer: increase directory listing pagination size

This commit is contained in:
Chris Lu 2019-12-12 09:11:10 -08:00
parent bf4b13612d
commit d0b423bbc0
5 changed files with 15 additions and 14 deletions

View file

@ -3,18 +3,22 @@ package filer2
import ( import (
"context" "context"
"fmt" "fmt"
"google.golang.org/grpc"
"math" "math"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
"google.golang.org/grpc"
"github.com/karlseguin/ccache"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/karlseguin/ccache"
) )
const PaginationSize = 1024 * 256
var ( var (
OS_UID = uint32(os.Getuid()) OS_UID = uint32(os.Getuid())
OS_GID = uint32(os.Getgid()) OS_GID = uint32(os.Getgid())
@ -32,7 +36,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
f := &Filer{ f := &Filer{
directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)), directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
MasterClient: wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters), MasterClient: wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
fileIdDeletionChan: make(chan string, 4096), fileIdDeletionChan: make(chan string, PaginationSize),
GrpcDialOption: grpcDialOption, GrpcDialOption: grpcDialOption,
} }
@ -217,7 +221,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
lastFileName := "" lastFileName := ""
includeLastFile := false includeLastFile := false
for limit > 0 { for limit > 0 {
entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024) entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, PaginationSize)
if err != nil { if err != nil {
glog.Errorf("list folder %s: %v", p, err) glog.Errorf("list folder %s: %v", p, err)
return fmt.Errorf("list folder %s: %v", p, err) return fmt.Errorf("list folder %s: %v", p, err)
@ -241,7 +245,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
} }
} }
if len(entries) < 1024 { if len(entries) < PaginationSize {
break break
} }
} }

View file

@ -128,8 +128,6 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error { err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
paginationLimit := 1024 * 256
lastEntryName := "" lastEntryName := ""
for { for {
@ -137,7 +135,7 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
request := &filer_pb.ListEntriesRequest{ request := &filer_pb.ListEntriesRequest{
Directory: fullDirPath, Directory: fullDirPath,
StartFromFileName: lastEntryName, StartFromFileName: lastEntryName,
Limit: uint32(paginationLimit), Limit: PaginationSize,
} }
glog.V(3).Infof("read directory: %v", request) glog.V(3).Infof("read directory: %v", request)
@ -151,7 +149,7 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
lastEntryName = entry.Name lastEntryName = entry.Name
} }
if len(resp.Entries) < paginationLimit { if len(resp.Entries) < PaginationSize {
break break
} }

View file

@ -214,7 +214,6 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error { err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
paginationLimit := 1024 * 256
remaining := dir.wfs.option.DirListingLimit remaining := dir.wfs.option.DirListingLimit
lastEntryName := "" lastEntryName := ""
@ -224,7 +223,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
request := &filer_pb.ListEntriesRequest{ request := &filer_pb.ListEntriesRequest{
Directory: dir.Path, Directory: dir.Path,
StartFromFileName: lastEntryName, StartFromFileName: lastEntryName,
Limit: uint32(paginationLimit), Limit: filer2.PaginationSize,
} }
glog.V(4).Infof("read directory: %v", request) glog.V(4).Infof("read directory: %v", request)
@ -250,7 +249,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
remaining -= len(resp.Entries) remaining -= len(resp.Entries)
if len(resp.Entries) < paginationLimit { if len(resp.Entries) < filer2.PaginationSize {
break break
} }

View file

@ -107,7 +107,7 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str
ret = append(ret, result...) ret = append(ret, result...)
} }
glog.V(0).Infof("deleted %d items", len(ret)) glog.V(1).Infof("deleted %d items", len(ret))
return ret, err return ret, err
} }

View file

@ -40,7 +40,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
limit = fs.option.DirListingLimit limit = fs.option.DirListingLimit
} }
paginationLimit := 1024 * 256 paginationLimit := filer2.PaginationSize
if limit < paginationLimit { if limit < paginationLimit {
paginationLimit = limit paginationLimit = limit
} }