gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Refactoring for supporing cassandra as filer meta data store

Browse source
This commit is contained in:
Chris Lu 2015-01-05 14:58:30 -08:00
parent a3e4145e8a
commit 165734ce11
8 changed files with 53 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
go/filer/directory.go
View file

@ -1,18 +0,0 @@
package filer
type DirectoryId int32
type DirectoryEntry struct {
Name string //dir name without path
Id DirectoryId
}
type DirectoryManager interface {
FindDirectory(dirPath string) (DirectoryId, error)
ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error)
MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
DeleteDirectory(dirPath string) error
//functions used by FUSE
FindDirectoryById(DirectoryId, error)
}

15
go/filer/embedded_filer/directory.go Normal file
View file

@ -0,0 +1,15 @@
package embedded_filer
import (
"github.com/chrislusf/weed-fs/go/filer"
)
type DirectoryManager interface {
FindDirectory(dirPath string) (filer.DirectoryId, error)
ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
DeleteDirectory(dirPath string) error
//functions used by FUSE
FindDirectoryById(filer.DirectoryId, error)
}

19
go/filer/directory_in_map.go → go/filer/embedded_filer/directory_in_map.go
View file

@ -1,4 +1,4 @@
package filer
package embedded_filer
import (
"bufio"
@ -10,6 +10,7 @@ import (
"strings"
"sync"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/util"
)
@ -19,12 +20,12 @@ type DirectoryEntryInMap struct {
Name string
Parent *DirectoryEntryInMap
SubDirectories map[string]*DirectoryEntryInMap
Id DirectoryId
Id filer.DirectoryId
}
type DirectoryManagerInMap struct {
Root *DirectoryEntryInMap
max DirectoryId
max filer.DirectoryId
logFile *os.File
isLoading bool
}
@ -83,7 +84,7 @@ func (dm *DirectoryManagerInMap) processEachLine(line string) error {
if pe != nil {
return pe
}
if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil {
if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
return e
}
case "mov":
@ -142,7 +143,7 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI
}
return dir, nil
}
func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, error) {
func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
d, e := dm.findDirectory(dirPath)
if e == nil {
return d.Id, nil
@ -150,7 +151,7 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, err
return dm.Root.Id, e
}
func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error {
func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
dirPath = filepath.Clean(dirPath)
if dirPath == "/" {
return nil
@ -201,7 +202,7 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn
return dir, created
}
func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) {
func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
dir, _ := dm.makeDirectory(dirPath)
return dir.Id, nil
}
@ -228,13 +229,13 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent
return nil
}
func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []DirectoryEntry, err error) {
func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
d, e := dm.findDirectory(dirPath)
if e != nil {
return dirNames, e
}
for k, v := range d.SubDirectories {
dirNames = append(dirNames, DirectoryEntry{Name: k, Id: v.Id})
dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
}
return dirNames, nil
}

2
go/filer/directory_test.go → go/filer/embedded_filer/directory_test.go
View file

@ -1,4 +1,4 @@
package filer
package embedded_filer
import (
"os"

9
go/filer/filer_embedded.go → go/filer/embedded_filer/filer_embedded.go
View file

@ -1,4 +1,4 @@
package filer
package embedded_filer
import (
"errors"
@ -6,6 +6,7 @@ import (
"path/filepath"
"strings"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/operation"
)
@ -48,13 +49,13 @@ func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
}
return filer.files.FindFile(dirId, file)
}
func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId DirectoryId, err error) {
func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
return filer.directories.FindDirectory(dirPath)
}
func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) {
func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
return filer.directories.ListDirectories(dirPath)
}
func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) {
func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
dirId, e := filer.directories.FindDirectory(dirPath)
if e != nil {
return nil, e

15
go/filer/files_in_leveldb.go → go/filer/embedded_filer/files_in_leveldb.go
View file

@ -1,8 +1,9 @@
package filer
package embedded_filer
import (
"bytes"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
@ -27,7 +28,7 @@ func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
return
}
func genKey(dirId DirectoryId, fileName string) []byte {
func genKey(dirId filer.DirectoryId, fileName string) []byte {
ret := make([]byte, 0, 4+len(fileName))
for i := 3; i >= 0; i-- {
ret = append(ret, byte(dirId>>(uint(i)*8)))
@ -36,25 +37,25 @@ func genKey(dirId DirectoryId, fileName string) []byte {
return ret
}
func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) {
func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
}
func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) {
func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
if fid, err = fl.FindFile(dirId, fileName); err != nil {
return
}
err = fl.db.Delete(genKey(dirId, fileName), nil)
return fid, err
}
func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) {
func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
data, e := fl.db.Get(genKey(dirId, fileName), nil)
if e != nil {
return "", e
}
return string(data), nil
}
func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []FileEntry) {
func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
dirKey := genKey(dirId, "")
iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
@ -74,7 +75,7 @@ func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, l
break
}
}
files = append(files, FileEntry{Name: fileName, Id: FileId(string(iter.Value()))})
files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))})
}
iter.Release()
return

15
go/filer/filer.go
View file

@ -7,13 +7,22 @@ type FileEntry struct {
Id FileId `json:"fid,omitempty"`
}
type DirectoryId int32
type DirectoryEntry struct {
Name string //dir name without path
Id DirectoryId
}
type Filer interface {
CreateFile(filePath string, fid string) (err error)
FindFile(filePath string) (fid string, err error)
CreateFile(fullFileName string, fid string) (err error)
FindFile(fullFileName string) (fid string, err error)
DeleteFile(fullFileName string) (fid string, err error)
//Optional functions. embedded filer support these
FindDirectory(dirPath string) (dirId DirectoryId, err error)
ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
DeleteDirectory(dirPath string, recursive bool) (err error)
DeleteFile(filePath string) (fid string, err error)
Move(fromPath string, toPath string) (err error)
}

3
go/weed/weed_server/filer_server.go
View file

@ -5,6 +5,7 @@ import (
"strconv"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/filer/embedded_filer"
"github.com/chrislusf/weed-fs/go/glog"
)
@ -28,7 +29,7 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle
port: ":" + strconv.Itoa(port),
}
if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil {
if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil {
glog.Fatal("Can not start filer in dir", dir, ": ", err.Error())
return
}