just refactoring

This commit is contained in:
Chris Lu 2020-11-26 15:08:05 -08:00
parent b11449f955
commit 97f706f35b

View file

@ -40,11 +40,11 @@ func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32
idxDir = util.ResolvePath(idxDir) idxDir = util.ResolvePath(idxDir)
} }
location := &DiskLocation{ location := &DiskLocation{
Directory: dir, Directory: dir,
IdxDirectory: idxDir, IdxDirectory: idxDir,
MaxVolumeCount: maxVolumeCount, MaxVolumeCount: maxVolumeCount,
OriginalMaxVolumeCount: maxVolumeCount, OriginalMaxVolumeCount: maxVolumeCount,
MinFreeSpacePercent: minFreeSpacePercent, MinFreeSpacePercent: minFreeSpacePercent,
} }
location.volumes = make(map[needle.VolumeId]*Volume) location.volumes = make(map[needle.VolumeId]*Volume)
location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
@ -74,53 +74,56 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI
func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool { func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool {
name := fileInfo.Name() name := fileInfo.Name()
if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") { if fileInfo.IsDir() {
name := name[:len(name)-len(".idx")] return false
noteFile := l.Directory + "/" + name + ".note" }
if util.FileExists(noteFile) { if !strings.HasSuffix(name, ".idx") {
note, _ := ioutil.ReadFile(noteFile) return false
glog.Warningf("volume %s was not completed: %s", name, string(note)) }
removeVolumeFiles(l.Directory + "/" + name) name = name[:len(name)-len(".idx")]
return false noteFile := l.Directory + "/" + name + ".note"
} if util.FileExists(noteFile) {
vid, collection, err := l.volumeIdFromPath(fileInfo) note, _ := ioutil.ReadFile(noteFile)
if err != nil { glog.Warningf("volume %s was not completed: %s", name, string(note))
glog.Warningf("get volume id failed, %s, err : %s", name, err) removeVolumeFiles(l.Directory + "/" + name)
return false return false
} }
vid, collection, err := l.volumeIdFromPath(fileInfo)
if err != nil {
glog.Warningf("get volume id failed, %s, err : %s", name, err)
return false
}
// void loading one volume more than once // void loading one volume more than once
l.volumesLock.RLock() l.volumesLock.RLock()
_, found := l.volumes[vid] _, found := l.volumes[vid]
l.volumesLock.RUnlock() l.volumesLock.RUnlock()
if found { if found {
glog.V(1).Infof("loaded volume, %v", vid) glog.V(1).Infof("loaded volume, %v", vid)
return true
}
v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, 0)
if e != nil {
glog.V(0).Infof("new volume %s error %s", name, e)
return false
}
l.SetVolume(vid, v)
size, _, _ := v.FileStat()
glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
l.Directory+"/"+name+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
return true return true
} }
return false
v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, 0)
if e != nil {
glog.V(0).Infof("new volume %s error %s", name, e)
return false
}
l.SetVolume(vid, v)
size, _, _ := v.FileStat()
glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
l.Directory+"/"+name+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
return true
} }
func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrency int) { func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrency int) {
task_queue := make(chan os.FileInfo, 10*concurrency) task_queue := make(chan os.FileInfo, 10*concurrency)
go func() { go func() {
if dirs, err := ioutil.ReadDir(l.Directory); err == nil { if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
for _, dir := range dirs { for _, fi := range fileInfos {
task_queue <- dir task_queue <- fi
} }
} }
close(task_queue) close(task_queue)
@ -131,8 +134,8 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
for dir := range task_queue { for fi := range task_queue {
_ = l.loadExistingVolume(dir, needleMapKind) _ = l.loadExistingVolume(fi, needleMapKind)
} }
}() }()
} }