From 28d4e1a51ba624fd4f5dbfbf386a6301de64b559 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 13 Oct 2020 19:49:52 -0700
Subject: [PATCH] mount: retry for directory listing with filer

related to https://github.com/chrislusf/seaweedfs/issues/1530
---
 weed/filesys/meta_cache/meta_cache_init.go | 31 +++++++++++++++-------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/weed/filesys/meta_cache/meta_cache_init.go b/weed/filesys/meta_cache/meta_cache_init.go
index 3e1719224..f42d61230 100644
--- a/weed/filesys/meta_cache/meta_cache_init.go
+++ b/weed/filesys/meta_cache/meta_cache_init.go
@@ -3,6 +3,8 @@ package meta_cache
 import (
 	"context"
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -16,19 +18,28 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full
 
 		glog.V(4).Infof("ReadDirAllEntries %s ...", path)
 
-		err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
-			entry := filer.FromPbEntry(string(dirPath), pbEntry)
-			if err := mc.doInsertEntry(context.Background(), entry); err != nil {
-				glog.V(0).Infof("read %s: %v", entry.FullPath, err)
-				return err
+		for waitTime := time.Second; waitTime < filer.ReadWaitTime; waitTime += waitTime / 2 {
+			err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
+				entry := filer.FromPbEntry(string(dirPath), pbEntry)
+				if err := mc.doInsertEntry(context.Background(), entry); err != nil {
+					glog.V(0).Infof("read %s: %v", entry.FullPath, err)
+					return err
+				}
+				if entry.IsDirectory() {
+					childDirectories = append(childDirectories, entry.Name())
+				}
+				return nil
+			})
+			if err == nil {
+				break
 			}
-			if entry.IsDirectory() {
-				childDirectories = append(childDirectories, entry.Name())
+			if strings.Contains(err.Error(), "transport: ") {
+				glog.V(0).Infof("ReadDirAllEntries %s: %v. Retry in %v", path, err, waitTime)
+				time.Sleep(waitTime)
+				continue
 			}
-			return nil
-		})
-		if err != nil {
 			err = fmt.Errorf("list %s: %v", dirPath, err)
+			break
 		}
 		return
 	})