refactor concurrent loading code logic

weed/storage/disk_location.go

@@ -22,28 +22,7 @@ func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
 	return location
 }
 
-func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
+func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleMapType, mutex *sync.RWMutex) {
-	l.Lock()
-	defer l.Unlock()
-
-	task_queue := make(chan os.FileInfo, 100)
-	go func() {
-		if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
-			for _, dir := range dirs {
-				task_queue <- dir
-			}
-		}
-		close(task_queue)
-	}()
-
-	const concurrency int = 10
-	var wg sync.WaitGroup
-	var mutex sync.RWMutex
-	for workerNum := 0; workerNum < concurrency; workerNum++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for dir := range task_queue {
 	name := dir.Name()
 	if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
 		collection := ""
@@ -69,10 +48,47 @@ func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
 		}
 	}
 }
+
+func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrentFlag bool) {
+	var concurrency int
+	if concurrentFlag {
+		//You could choose a better optimized concurency value after testing at your environment
+		concurrency = 10
+	} else {
+		concurrency = 1
+	}
+
+	task_queue := make(chan os.FileInfo, 10*concurrency)
+	go func() {
+		if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
+			for _, dir := range dirs {
+				task_queue <- dir
+			}
+		}
+		close(task_queue)
+	}()
+
+	var wg sync.WaitGroup
+	var mutex sync.RWMutex
+	for workerNum := 0; workerNum < concurrency; workerNum++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for dir := range task_queue {
+				l.loadExistingVolume(dir, needleMapKind, &mutex)
+			}
 		}()
 	}
 	wg.Wait()
 
+}
+
+func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
+	l.Lock()
+	defer l.Unlock()
+
+	l.concurrentLoadingVolumes(needleMapKind, true)
+
 	glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
 }