gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

batch file id deletion

Browse source
This commit is contained in:
Chris Lu 2020-05-10 23:25:39 -07:00
parent 20b8f2e5b4
commit 015dd3a147
1 changed files with 21 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
weed/filer2/filer_deletion.go
View file

@ -34,19 +34,31 @@ func (f *Filer) loopProcessingDeletion() {
lookupFunc := LookupByMasterClientFn(f.MasterClient) lookupFunc := LookupByMasterClientFn(f.MasterClient)
DeletionBatchSize := 100000 // roughly 20 bytes cost per file id.
var deletionCount int var deletionCount int
for { for {
deletionCount = 0 deletionCount = 0
f.fileIdDeletionQueue.Consume(func(fileIds []string) { f.fileIdDeletionQueue.Consume(func(fileIds []string) {
deletionCount = len(fileIds) for len(fileIds) > 0 {
deleteResults, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc) var toDeleteFileIds []string
if err != nil { if len(fileIds) > DeletionBatchSize {
glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err) toDeleteFileIds = fileIds[:DeletionBatchSize]
} else { fileIds = fileIds[DeletionBatchSize:]
glog.V(1).Infof("deleting fileIds len=%d", deletionCount) } else {
} toDeleteFileIds = fileIds
if len(deleteResults) != deletionCount { fileIds = fileIds[:0]
glog.V(0).Infof("delete %d fileIds actual %d", deletionCount, len(deleteResults)) }
deletionCount = len(toDeleteFileIds)
deleteResults, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc)
if err != nil {
glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err)
} else {
glog.V(1).Infof("deleting fileIds len=%d", deletionCount)
}
if len(deleteResults) != deletionCount {
glog.V(0).Infof("delete %d fileIds actual %d", deletionCount, len(deleteResults))
}
} }
}) })