From 3d8df0f709795a3e0e9ba6849dcc7b7614173ca9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 2 Jul 2016 23:56:49 -0700 Subject: [PATCH] refactor volume_loading.go out of volume.go --- weed/storage/volume.go | 79 +----------------------------- weed/storage/volume_loading.go | 89 ++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 78 deletions(-) create mode 100644 weed/storage/volume_loading.go diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 5ec441f05..2053cd679 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -39,13 +39,6 @@ func (v *Volume) String() string { return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, readOnly:%v", v.Id, v.dir, v.Collection, v.dataFile, v.nm, v.readOnly) } -func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType) (v *Volume, e error) { - v = &Volume{dir: dirname, Collection: collection, Id: id} - v.SuperBlock = SuperBlock{} - v.needleMapKind = needleMapKind - e = v.load(false, false, needleMapKind) - return -} func (v *Volume) FileName() (fileName string) { if v.Collection == "" { fileName = path.Join(v.dir, v.Id.String()) @@ -57,81 +50,11 @@ func (v *Volume) FileName() (fileName string) { func (v *Volume) DataFile() *os.File { return v.dataFile } -func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error { - var e error - fileName := v.FileName() - if exists, canRead, canWrite, modifiedTime := checkFile(fileName + ".dat"); exists { - if !canRead { - return fmt.Errorf("cannot read Volume Data file %s.dat", fileName) - } - if canWrite { - v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) - v.lastModifiedTime = uint64(modifiedTime.Unix()) - } else { - glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode") - v.dataFile, e = os.Open(fileName + ".dat") - v.readOnly = true - } - } else { - if createDatIfMissing { - v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) - } else { - return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName) - } - } - - if e != nil { - if !os.IsPermission(e) { - return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e) - } - } - - if v.ReplicaPlacement == nil { - e = v.readSuperBlock() - } else { - e = v.maybeWriteSuperBlock() - } - if e == nil && alsoLoadIndex { - var indexFile *os.File - if v.readOnly { - glog.V(1).Infoln("open to read file", fileName+".idx") - if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil { - return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e) - } - } else { - glog.V(1).Infoln("open to write file", fileName+".idx") - if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil { - return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e) - } - } - if e = CheckVolumeDataIntegrity(v, indexFile); e != nil { - v.readOnly = true - glog.V(0).Infof("volumeDataIntegrityChecking failed %v", e) - } - switch needleMapKind { - case NeedleMapInMemory: - glog.V(0).Infoln("loading index file", fileName+".idx", "readonly", v.readOnly) - if v.nm, e = LoadNeedleMap(indexFile); e != nil { - glog.V(0).Infof("loading index %s error: %v", fileName+".idx", e) - } - case NeedleMapLevelDb: - glog.V(0).Infoln("loading leveldb file", fileName+".ldb") - if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil { - glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e) - } - case NeedleMapBoltDb: - glog.V(0).Infoln("loading boltdb file", fileName+".bdb") - if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil { - glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e) - } - } - } - return e -} func (v *Volume) Version() Version { return v.SuperBlock.Version() } + func (v *Volume) Size() int64 { stat, e := v.dataFile.Stat() if e == nil { diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go new file mode 100644 index 000000000..968471620 --- /dev/null +++ b/weed/storage/volume_loading.go @@ -0,0 +1,89 @@ +package storage + +import ( + "fmt" + "os" + + "github.com/chrislusf/seaweedfs/weed/glog" +) + +func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType) (v *Volume, e error) { + v = &Volume{dir: dirname, Collection: collection, Id: id} + v.SuperBlock = SuperBlock{} + v.needleMapKind = needleMapKind + e = v.load(false, false, needleMapKind) + return +} + +func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error { + var e error + fileName := v.FileName() + + if exists, canRead, canWrite, modifiedTime := checkFile(fileName + ".dat"); exists { + if !canRead { + return fmt.Errorf("cannot read Volume Data file %s.dat", fileName) + } + if canWrite { + v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) + v.lastModifiedTime = uint64(modifiedTime.Unix()) + } else { + glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode") + v.dataFile, e = os.Open(fileName + ".dat") + v.readOnly = true + } + } else { + if createDatIfMissing { + v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) + } else { + return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName) + } + } + + if e != nil { + if !os.IsPermission(e) { + return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e) + } + } + + if v.ReplicaPlacement == nil { + e = v.readSuperBlock() + } else { + e = v.maybeWriteSuperBlock() + } + if e == nil && alsoLoadIndex { + var indexFile *os.File + if v.readOnly { + glog.V(1).Infoln("open to read file", fileName+".idx") + if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil { + return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e) + } + } else { + glog.V(1).Infoln("open to write file", fileName+".idx") + if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil { + return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e) + } + } + if e = CheckVolumeDataIntegrity(v, indexFile); e != nil { + v.readOnly = true + glog.V(0).Infof("volumeDataIntegrityChecking failed %v", e) + } + switch needleMapKind { + case NeedleMapInMemory: + glog.V(0).Infoln("loading index file", fileName+".idx", "readonly", v.readOnly) + if v.nm, e = LoadNeedleMap(indexFile); e != nil { + glog.V(0).Infof("loading index %s error: %v", fileName+".idx", e) + } + case NeedleMapLevelDb: + glog.V(0).Infoln("loading leveldb file", fileName+".ldb") + if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil { + glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e) + } + case NeedleMapBoltDb: + glog.V(0).Infoln("loading boltdb file", fileName+".bdb") + if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil { + glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e) + } + } + } + return e +}