mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
removing filer package!
This commit is contained in:
Chris Lu
parent
03d76479a5
commit
8647191bee
|
|
@ -1,96 +0,0 @@
|
|||
package cassandra_store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
||||
"github.com/gocql/gocql"
|
||||
)
|
||||
|
||||
/*
|
||||
|
||||
Basically you need a table just like this:
|
||||
|
||||
CREATE TABLE seaweed_files (
|
||||
path varchar,
|
||||
fids list<varchar>,
|
||||
PRIMARY KEY (path)
|
||||
);
|
||||
|
||||
Need to match flat_namespace.FlatNamespaceStore interface
|
||||
Put(fullFileName string, fid string) (err error)
|
||||
Get(fullFileName string) (fid string, err error)
|
||||
Delete(fullFileName string) (fid string, err error)
|
||||
|
||||
*/
|
||||
type CassandraStore struct {
|
||||
cluster *gocql.ClusterConfig
|
||||
session *gocql.Session
|
||||
}
|
||||
|
||||
func NewCassandraStore(keyspace string, hosts string) (c *CassandraStore, err error) {
|
||||
c = &CassandraStore{}
|
||||
s := strings.Split(hosts, ",")
|
||||
if len(s) == 1 {
|
||||
glog.V(2).Info("Only one cassandra node to connect! A cluster is Recommended! Now using:", string(hosts))
|
||||
c.cluster = gocql.NewCluster(hosts)
|
||||
} else if len(s) > 1 {
|
||||
c.cluster = gocql.NewCluster(s...)
|
||||
}
|
||||
c.cluster.Keyspace = keyspace
|
||||
c.cluster.Consistency = gocql.LocalQuorum
|
||||
c.session, err = c.cluster.CreateSession()
|
||||
if err != nil {
|
||||
glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *CassandraStore) Put(fullFileName string, fid string) (err error) {
|
||||
var input []string
|
||||
input = append(input, fid)
|
||||
if err := c.session.Query(
|
||||
`INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`,
|
||||
fullFileName, input).Exec(); err != nil {
|
||||
glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
|
||||
var output []string
|
||||
if err := c.session.Query(
|
||||
`select fids FROM seaweed_files WHERE path = ? LIMIT 1`,
|
||||
fullFileName).Consistency(gocql.One).Scan(&output); err != nil {
|
||||
if err != gocql.ErrNotFound {
|
||||
glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err)
|
||||
return "", filer.ErrNotFound
|
||||
}
|
||||
}
|
||||
if len(output) == 0 {
|
||||
return "", fmt.Errorf("No file id found for %s", fullFileName)
|
||||
}
|
||||
return output[0], nil
|
||||
}
|
||||
|
||||
// Currently the fid is not returned
|
||||
func (c *CassandraStore) Delete(fullFileName string) (err error) {
|
||||
if err := c.session.Query(
|
||||
`DELETE FROM seaweed_files WHERE path = ?`,
|
||||
fullFileName).Exec(); err != nil {
|
||||
if err != gocql.ErrNotFound {
|
||||
glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CassandraStore) Close() {
|
||||
if c.session != nil {
|
||||
c.session.Close()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
/*
|
||||
|
||||
Here is the CQL to create the table.CassandraStore
|
||||
|
||||
Optionally you can adjust the keyspace name and replication settings.
|
||||
|
||||
For production server, very likely you want to set replication_factor to 3
|
||||
|
||||
*/
|
||||
|
||||
create keyspace seaweed WITH replication = {
|
||||
'class':'SimpleStrategy',
|
||||
'replication_factor':1
|
||||
};
|
||||
|
||||
use seaweed;
|
||||
|
||||
CREATE TABLE seaweed_files (
|
||||
path varchar,
|
||||
fids list<varchar>,
|
||||
PRIMARY KEY (path)
|
||||
);
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
Design Assumptions:
|
||||
1. the number of directories are magnitudely smaller than the number of files
|
||||
2. unlimited number of files under any directories
|
||||
Phylosophy:
|
||||
metadata for directories and files should be separated
|
||||
Design:
|
||||
Store directories in normal map
|
||||
all of directories hopefully all be in memory
|
||||
efficient to move/rename/list_directories
|
||||
Log directory changes to append only log file
|
||||
Store files in sorted string table in <dir_id/filename> format
|
||||
efficient to list_files, just simple iterator
|
||||
efficient to locate files, binary search
|
||||
|
||||
Testing:
|
||||
1. starting server, "weed server -filer=true"
|
||||
2. posting files to different folders
|
||||
curl -F "filename=@design.txt" "http://localhost:8888/sources/"
|
||||
curl -F "filename=@design.txt" "http://localhost:8888/design/"
|
||||
curl -F "filename=@directory.go" "http://localhost:8888/sources/weed/go/"
|
||||
curl -F "filename=@directory.go" "http://localhost:8888/sources/testing/go/"
|
||||
curl -F "filename=@filer.go" "http://localhost:8888/sources/weed/go/"
|
||||
curl -F "filename=@filer_in_leveldb.go" "http://localhost:8888/sources/weed/go/"
|
||||
curl "http://localhost:8888/?pretty=y"
|
||||
curl "http://localhost:8888/sources/weed/go/?pretty=y"
|
||||
curl "http://localhost:8888/sources/weed/go/?pretty=y"
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
package embedded_filer
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
)
|
||||
|
||||
type DirectoryManager interface {
|
||||
FindDirectory(dirPath string) (DirectoryId, error)
|
||||
ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error)
|
||||
MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error)
|
||||
MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
|
||||
DeleteDirectory(dirPath string) error
|
||||
//functions used by FUSE
|
||||
FindDirectoryById(DirectoryId, error)
|
||||
}
|
||||
|
|
@ -1,312 +0,0 @@
|
|||
package embedded_filer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var writeLock sync.Mutex //serialize changes to dir.log
|
||||
|
||||
type DirectoryId int32
|
||||
|
||||
type DirectoryEntryInMap struct {
|
||||
sync.Mutex
|
||||
Name string
|
||||
Parent *DirectoryEntryInMap
|
||||
subDirectories map[string]*DirectoryEntryInMap
|
||||
Id DirectoryId
|
||||
}
|
||||
|
||||
func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
|
||||
de.Lock()
|
||||
defer de.Unlock()
|
||||
child, ok := de.subDirectories[dirName]
|
||||
return child, ok
|
||||
}
|
||||
func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) {
|
||||
de.Lock()
|
||||
defer de.Unlock()
|
||||
de.subDirectories[dirName] = child
|
||||
}
|
||||
func (de *DirectoryEntryInMap) removeChild(dirName string) {
|
||||
de.Lock()
|
||||
defer de.Unlock()
|
||||
delete(de.subDirectories, dirName)
|
||||
}
|
||||
func (de *DirectoryEntryInMap) hasChildren() bool {
|
||||
de.Lock()
|
||||
defer de.Unlock()
|
||||
return len(de.subDirectories) > 0
|
||||
}
|
||||
func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryName) {
|
||||
de.Lock()
|
||||
defer de.Unlock()
|
||||
for k, _ := range de.subDirectories {
|
||||
dirNames = append(dirNames, filer.DirectoryName(k))
|
||||
}
|
||||
return dirNames
|
||||
}
|
||||
|
||||
type DirectoryManagerInMap struct {
|
||||
Root *DirectoryEntryInMap
|
||||
max DirectoryId
|
||||
logFile *os.File
|
||||
isLoading bool
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) newDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
|
||||
d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)}
|
||||
var parts []string
|
||||
for p := d; p != nil && p.Name != ""; p = p.Parent {
|
||||
parts = append(parts, p.Name)
|
||||
}
|
||||
n := len(parts)
|
||||
if n <= 0 {
|
||||
return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name)
|
||||
}
|
||||
for i := 0; i < n/2; i++ {
|
||||
parts[i], parts[n-1-i] = parts[n-1-i], parts[i]
|
||||
}
|
||||
dm.max++
|
||||
d.Id = dm.max
|
||||
dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id)))
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) log(words ...string) {
|
||||
if !dm.isLoading {
|
||||
dm.logFile.WriteString(strings.Join(words, "\t") + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
|
||||
dm = &DirectoryManagerInMap{}
|
||||
//dm.Root do not use newDirectoryEntryInMap, since dm.max will be changed
|
||||
dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)}
|
||||
if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
|
||||
return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err)
|
||||
}
|
||||
return dm, dm.load()
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) processEachLine(line string) error {
|
||||
if strings.HasPrefix(line, "#") {
|
||||
return nil
|
||||
}
|
||||
if line == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(line, "\t")
|
||||
if len(parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
switch parts[0] {
|
||||
case "add":
|
||||
v, pe := strconv.Atoi(parts[2])
|
||||
if pe != nil {
|
||||
return pe
|
||||
}
|
||||
if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil {
|
||||
return e
|
||||
}
|
||||
case "mov":
|
||||
newName := ""
|
||||
if len(parts) >= 4 {
|
||||
newName = parts[3]
|
||||
}
|
||||
if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil {
|
||||
return e
|
||||
}
|
||||
case "del":
|
||||
if e := dm.DeleteDirectory(parts[1]); e != nil {
|
||||
return e
|
||||
}
|
||||
default:
|
||||
fmt.Printf("line %s has %s!\n", line, parts[0])
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (dm *DirectoryManagerInMap) load() error {
|
||||
dm.max = 0
|
||||
lines := bufio.NewReader(dm.logFile)
|
||||
dm.isLoading = true
|
||||
defer func() { dm.isLoading = false }()
|
||||
for {
|
||||
line, err := util.Readln(lines)
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
if pe := dm.processEachLine(string(line)); pe != nil {
|
||||
return pe
|
||||
}
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) {
|
||||
if dirPath == "" {
|
||||
return dm.Root, nil
|
||||
}
|
||||
dirPath = CleanFilePath(dirPath)
|
||||
if dirPath == "/" {
|
||||
return dm.Root, nil
|
||||
}
|
||||
parts := strings.Split(dirPath, "/")
|
||||
dir := dm.Root
|
||||
for i := 1; i < len(parts); i++ {
|
||||
if sub, ok := dir.getChild(parts[i]); ok {
|
||||
dir = sub
|
||||
} else {
|
||||
return dm.Root, filer.ErrNotFound
|
||||
}
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
func (dm *DirectoryManagerInMap) findDirectoryId(dirPath string) (DirectoryId, error) {
|
||||
d, e := dm.findDirectory(dirPath)
|
||||
if e == nil {
|
||||
return d.Id, nil
|
||||
}
|
||||
return dm.Root.Id, e
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error {
|
||||
dirPath = CleanFilePath(dirPath)
|
||||
if dirPath == "/" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(dirPath, "/")
|
||||
dir := dm.Root
|
||||
for i := 1; i < len(parts); i++ {
|
||||
sub, ok := dir.getChild(parts[i])
|
||||
if !ok {
|
||||
writeLock.Lock()
|
||||
if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
|
||||
sub = sub2
|
||||
} else {
|
||||
if i != len(parts)-1 {
|
||||
writeLock.Unlock()
|
||||
return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
|
||||
}
|
||||
var err error
|
||||
sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
|
||||
if err != nil {
|
||||
writeLock.Unlock()
|
||||
return err
|
||||
}
|
||||
if sub.Id != dirId {
|
||||
writeLock.Unlock()
|
||||
// the dir.log should be the same order as in-memory directory id
|
||||
return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
|
||||
}
|
||||
dir.addChild(parts[i], sub)
|
||||
}
|
||||
writeLock.Unlock()
|
||||
}
|
||||
dir = sub
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
|
||||
dirPath = CleanFilePath(dirPath)
|
||||
if dirPath == "/" {
|
||||
return dm.Root, false
|
||||
}
|
||||
parts := strings.Split(dirPath, "/")
|
||||
dir = dm.Root
|
||||
for i := 1; i < len(parts); i++ {
|
||||
sub, ok := dir.getChild(parts[i])
|
||||
if !ok {
|
||||
writeLock.Lock()
|
||||
if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
|
||||
sub = sub2
|
||||
} else {
|
||||
var err error
|
||||
sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
|
||||
if err != nil {
|
||||
writeLock.Unlock()
|
||||
return nil, false
|
||||
}
|
||||
dir.addChild(parts[i], sub)
|
||||
created = true
|
||||
}
|
||||
writeLock.Unlock()
|
||||
}
|
||||
dir = sub
|
||||
}
|
||||
return dir, created
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) {
|
||||
dir, _ := dm.makeDirectory(dirPath)
|
||||
return dir.Id, nil
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error {
|
||||
writeLock.Lock()
|
||||
defer writeLock.Unlock()
|
||||
oldDir, oe := dm.findDirectory(oldDirPath)
|
||||
if oe != nil {
|
||||
return oe
|
||||
}
|
||||
parentDir, pe := dm.findDirectory(newParentDirPath)
|
||||
if pe != nil {
|
||||
return pe
|
||||
}
|
||||
dm.log("mov", oldDirPath, newParentDirPath, newName)
|
||||
oldDir.Parent.removeChild(oldDir.Name)
|
||||
if newName == "" {
|
||||
newName = oldDir.Name
|
||||
}
|
||||
parentDir.addChild(newName, oldDir)
|
||||
oldDir.Name = newName
|
||||
oldDir.Parent = parentDir
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryName, err error) {
|
||||
d, e := dm.findDirectory(dirPath)
|
||||
if e != nil {
|
||||
return dirNames, e
|
||||
}
|
||||
return d.children(), nil
|
||||
}
|
||||
func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
|
||||
writeLock.Lock()
|
||||
defer writeLock.Unlock()
|
||||
if dirPath == "/" {
|
||||
return fmt.Errorf("Can not delete %s", dirPath)
|
||||
}
|
||||
d, e := dm.findDirectory(dirPath)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if d.hasChildren() {
|
||||
return fmt.Errorf("dir %s still has sub directories", dirPath)
|
||||
}
|
||||
d.Parent.removeChild(d.Name)
|
||||
d.Parent = nil
|
||||
dm.log("del", dirPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func CleanFilePath(fp string) string {
|
||||
ret := filepath.Clean(fp)
|
||||
if os.PathSeparator == '\\' {
|
||||
return strings.Replace(ret, "\\", "/", -1)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
package embedded_filer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDirectory(t *testing.T) {
|
||||
dm, _ := NewDirectoryManagerInMap("/tmp/dir.log")
|
||||
defer func() {
|
||||
if true {
|
||||
os.Remove("/tmp/dir.log")
|
||||
}
|
||||
}()
|
||||
dm.MakeDirectory("/a/b/c")
|
||||
dm.MakeDirectory("/a/b/d")
|
||||
dm.MakeDirectory("/a/b/e")
|
||||
dm.MakeDirectory("/a/b/e/f")
|
||||
dm.MakeDirectory("/a/b/e/f/g")
|
||||
dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t")
|
||||
if _, err := dm.findDirectoryId("/a/b/e/f/g"); err == nil {
|
||||
t.Fatal("/a/b/e/f/g should not exist any more after moving")
|
||||
}
|
||||
if _, err := dm.findDirectoryId("/a/b/t"); err != nil {
|
||||
t.Fatal("/a/b/t should exist after moving")
|
||||
}
|
||||
if _, err := dm.findDirectoryId("/a/b/g"); err == nil {
|
||||
t.Fatal("/a/b/g should not exist after moving")
|
||||
}
|
||||
dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "")
|
||||
if _, err := dm.findDirectoryId("/a/b/f"); err != nil {
|
||||
t.Fatal("/a/b/g should not exist after moving")
|
||||
}
|
||||
dm.MakeDirectory("/a/b/g/h/i")
|
||||
dm.DeleteDirectory("/a/b/e/f")
|
||||
dm.DeleteDirectory("/a/b/e")
|
||||
dirNames, _ := dm.ListDirectories("/a/b/e")
|
||||
for _, v := range dirNames {
|
||||
println("sub1 dir:", v)
|
||||
}
|
||||
dm.logFile.Close()
|
||||
|
||||
var path []string
|
||||
printTree(dm.Root, path)
|
||||
|
||||
dm2, e := NewDirectoryManagerInMap("/tmp/dir.log")
|
||||
if e != nil {
|
||||
println("load error", e.Error())
|
||||
}
|
||||
if !compare(dm.Root, dm2.Root) {
|
||||
t.Fatal("restored dir not the same!")
|
||||
}
|
||||
printTree(dm2.Root, path)
|
||||
}
|
||||
|
||||
func printTree(node *DirectoryEntryInMap, path []string) {
|
||||
println(strings.Join(path, "/") + "/" + node.Name)
|
||||
path = append(path, node.Name)
|
||||
for _, v := range node.subDirectories {
|
||||
printTree(v, path)
|
||||
}
|
||||
}
|
||||
|
||||
func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool {
|
||||
if len(root1.subDirectories) != len(root2.subDirectories) {
|
||||
return false
|
||||
}
|
||||
if root1.Name != root2.Name {
|
||||
return false
|
||||
}
|
||||
if root1.Id != root2.Id {
|
||||
return false
|
||||
}
|
||||
if !(root1.Parent == nil && root2.Parent == nil) {
|
||||
if root1.Parent.Id != root2.Parent.Id {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for k, v := range root1.subDirectories {
|
||||
if !compare(v, root2.subDirectories[k]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -1,156 +0,0 @@
|
|||
package embedded_filer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
)
|
||||
|
||||
type FilerEmbedded struct {
|
||||
master string
|
||||
directories *DirectoryManagerInMap
|
||||
files *FileListInLevelDb
|
||||
mvMutex sync.Mutex
|
||||
}
|
||||
|
||||
func NewFilerEmbedded(master string, dir string) (filer *FilerEmbedded, err error) {
|
||||
dm, de := NewDirectoryManagerInMap(filepath.Join(dir, "dir.log"))
|
||||
if de != nil {
|
||||
return nil, de
|
||||
}
|
||||
fl, fe := NewFileListInLevelDb(dir)
|
||||
if fe != nil {
|
||||
return nil, fe
|
||||
}
|
||||
filer = &FilerEmbedded{
|
||||
master: master,
|
||||
directories: dm,
|
||||
files: fl,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error) {
|
||||
dir, file := filepath.Split(filePath)
|
||||
dirId, e := filer.directories.MakeDirectory(dir)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
return filer.files.CreateFile(dirId, file, fid)
|
||||
}
|
||||
func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
|
||||
dir, file := filepath.Split(filePath)
|
||||
return filer.findFileEntry(dir, file)
|
||||
}
|
||||
func (filer *FilerEmbedded) findFileEntry(parentPath string, fileName string) (fid string, err error) {
|
||||
dirId, e := filer.directories.findDirectoryId(parentPath)
|
||||
if e != nil {
|
||||
return "", e
|
||||
}
|
||||
return filer.files.FindFile(dirId, fileName)
|
||||
}
|
||||
|
||||
func (filer *FilerEmbedded) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
|
||||
if _, err = filer.directories.findDirectory(filepath.Join(dirPath, name)); err == nil {
|
||||
return true, "", nil
|
||||
}
|
||||
if fileId, err = filer.findFileEntry(dirPath, name); err == nil {
|
||||
return true, fileId, nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
|
||||
return filer.directories.ListDirectories(dirPath)
|
||||
}
|
||||
func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
|
||||
dirId, e := filer.directories.findDirectoryId(dirPath)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return filer.files.ListFiles(dirId, lastFileName, limit), nil
|
||||
}
|
||||
func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) {
|
||||
dirId, e := filer.directories.findDirectoryId(dirPath)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if sub_dirs, sub_err := filer.directories.ListDirectories(dirPath); sub_err == nil {
|
||||
if len(sub_dirs) > 0 && !recursive {
|
||||
return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs))
|
||||
}
|
||||
for _, sub := range sub_dirs {
|
||||
if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, string(sub)), recursive); delete_sub_err != nil {
|
||||
return delete_sub_err
|
||||
}
|
||||
}
|
||||
}
|
||||
list := filer.files.ListFiles(dirId, "", 100)
|
||||
if len(list) != 0 && !recursive {
|
||||
if !recursive {
|
||||
return fmt.Errorf("Fail to delete non-empty directory %s!", dirPath)
|
||||
}
|
||||
}
|
||||
for {
|
||||
if len(list) == 0 {
|
||||
return filer.directories.DeleteDirectory(dirPath)
|
||||
}
|
||||
var fids []string
|
||||
for _, fileEntry := range list {
|
||||
fids = append(fids, string(fileEntry.Id))
|
||||
}
|
||||
if result_list, delete_file_err := operation.DeleteFiles(filer.master, fids); delete_file_err != nil {
|
||||
return delete_file_err
|
||||
} else {
|
||||
if len(result_list.Errors) > 0 {
|
||||
return errors.New(strings.Join(result_list.Errors, "\n"))
|
||||
}
|
||||
}
|
||||
lastFile := list[len(list)-1]
|
||||
list = filer.files.ListFiles(dirId, lastFile.Name, 100)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) {
|
||||
dir, file := filepath.Split(filePath)
|
||||
dirId, e := filer.directories.findDirectoryId(dir)
|
||||
if e != nil {
|
||||
return "", e
|
||||
}
|
||||
return filer.files.DeleteFile(dirId, file)
|
||||
}
|
||||
|
||||
/*
|
||||
Move a folder or a file, with 4 Use cases:
|
||||
mv fromDir toNewDir
|
||||
mv fromDir toOldDir
|
||||
mv fromFile toDir
|
||||
mv fromFile toFile
|
||||
*/
|
||||
func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
|
||||
filer.mvMutex.Lock()
|
||||
defer filer.mvMutex.Unlock()
|
||||
|
||||
if _, dir_err := filer.directories.findDirectoryId(fromPath); dir_err == nil {
|
||||
if _, err := filer.directories.findDirectoryId(toPath); err == nil {
|
||||
// move folder under an existing folder
|
||||
return filer.directories.MoveUnderDirectory(fromPath, toPath, "")
|
||||
}
|
||||
// move folder to a new folder
|
||||
return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath))
|
||||
}
|
||||
if fid, file_err := filer.DeleteFile(fromPath); file_err == nil {
|
||||
if _, err := filer.directories.findDirectoryId(toPath); err == nil {
|
||||
// move file under an existing folder
|
||||
return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid)
|
||||
}
|
||||
// move to a folder with new name
|
||||
return filer.CreateFile(toPath, fid)
|
||||
}
|
||||
return fmt.Errorf("File %s is not found!", fromPath)
|
||||
}
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
package embedded_filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
/*
|
||||
The entry in level db has this format:
|
||||
key: genKey(dirId, fileName)
|
||||
value: []byte(fid)
|
||||
And genKey(dirId, fileName) use first 4 bytes to store dirId, and rest for fileName
|
||||
*/
|
||||
|
||||
type FileListInLevelDb struct {
|
||||
db *leveldb.DB
|
||||
}
|
||||
|
||||
func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
|
||||
fl = &FileListInLevelDb{}
|
||||
if fl.db, err = leveldb.OpenFile(dir, nil); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func genKey(dirId DirectoryId, fileName string) []byte {
|
||||
ret := make([]byte, 0, 4+len(fileName))
|
||||
for i := 3; i >= 0; i-- {
|
||||
ret = append(ret, byte(dirId>>(uint(i)*8)))
|
||||
}
|
||||
ret = append(ret, []byte(fileName)...)
|
||||
return ret
|
||||
}
|
||||
|
||||
func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) {
|
||||
glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
|
||||
return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
|
||||
}
|
||||
func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) {
|
||||
if fid, err = fl.FindFile(dirId, fileName); err != nil {
|
||||
if err == leveldb.ErrNotFound {
|
||||
return "", nil
|
||||
}
|
||||
return
|
||||
}
|
||||
err = fl.db.Delete(genKey(dirId, fileName), nil)
|
||||
return fid, err
|
||||
}
|
||||
func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) {
|
||||
data, e := fl.db.Get(genKey(dirId, fileName), nil)
|
||||
if e == leveldb.ErrNotFound {
|
||||
return "", filer.ErrNotFound
|
||||
} else if e != nil {
|
||||
return "", e
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
|
||||
glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
|
||||
dirKey := genKey(dirId, "")
|
||||
iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
|
||||
limitCounter := 0
|
||||
for iter.Next() {
|
||||
key := iter.Key()
|
||||
if !bytes.HasPrefix(key, dirKey) {
|
||||
break
|
||||
}
|
||||
fileName := string(key[len(dirKey):])
|
||||
if fileName == lastFileName {
|
||||
continue
|
||||
}
|
||||
limitCounter++
|
||||
if limit > 0 {
|
||||
if limitCounter > limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))})
|
||||
}
|
||||
iter.Release()
|
||||
return
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
type FileId string //file id in SeaweedFS
|
||||
|
||||
type FileEntry struct {
|
||||
Name string `json:"name,omitempty"` //file name without path
|
||||
Id FileId `json:"fid,omitempty"`
|
||||
}
|
||||
|
||||
type DirectoryName string
|
||||
|
||||
type Filer interface {
|
||||
CreateFile(fullFileName string, fid string) (err error)
|
||||
FindFile(fullFileName string) (fid string, err error)
|
||||
DeleteFile(fullFileName string) (fid string, err error)
|
||||
|
||||
//Optional functions. embedded filer support these
|
||||
ListDirectories(dirPath string) (dirs []DirectoryName, err error)
|
||||
ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
|
||||
DeleteDirectory(dirPath string, recursive bool) (err error)
|
||||
Move(fromPath string, toPath string) (err error)
|
||||
LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error)
|
||||
}
|
||||
|
||||
var ErrNotFound = errors.New("filer: no entry is found in filer store")
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
package flat_namespace
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type FlatNamespaceFiler struct {
|
||||
master string
|
||||
store FlatNamespaceStore
|
||||
}
|
||||
|
||||
var (
|
||||
ErrNotImplemented = errors.New("Not Implemented for flat namespace meta data store")
|
||||
)
|
||||
|
||||
func NewFlatNamespaceFiler(master string, store FlatNamespaceStore) *FlatNamespaceFiler {
|
||||
return &FlatNamespaceFiler{
|
||||
master: master,
|
||||
store: store,
|
||||
}
|
||||
}
|
||||
|
||||
func (filer *FlatNamespaceFiler) CreateFile(fullFileName string, fid string) (err error) {
|
||||
return filer.store.Put(fullFileName, fid)
|
||||
}
|
||||
func (filer *FlatNamespaceFiler) FindFile(fullFileName string) (fid string, err error) {
|
||||
return filer.store.Get(fullFileName)
|
||||
}
|
||||
func (filer *FlatNamespaceFiler) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
|
||||
if fileId, err = filer.FindFile(filepath.Join(dirPath, name)); err == nil {
|
||||
return true, fileId, nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
func (filer *FlatNamespaceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
func (filer *FlatNamespaceFiler) DeleteDirectory(dirPath string, recursive bool) (err error) {
|
||||
return ErrNotImplemented
|
||||
}
|
||||
|
||||
func (filer *FlatNamespaceFiler) DeleteFile(fullFileName string) (fid string, err error) {
|
||||
fid, err = filer.FindFile(fullFileName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = filer.store.Delete(fullFileName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fid, nil
|
||||
//return filer.store.Delete(fullFileName)
|
||||
//are you kidding me!!!!
|
||||
}
|
||||
|
||||
func (filer *FlatNamespaceFiler) Move(fromPath string, toPath string) error {
|
||||
return ErrNotImplemented
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
package flat_namespace
|
||||
|
||||
import ()
|
||||
|
||||
type FlatNamespaceStore interface {
|
||||
Put(fullFileName string, fid string) (err error)
|
||||
Get(fullFileName string) (fid string, err error)
|
||||
Delete(fullFileName string) (err error)
|
||||
}
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
#MySQL filer mapping store
|
||||
|
||||
## Schema format
|
||||
|
||||
|
||||
Basically, uriPath and fid are the key elements stored in MySQL. In view of the optimization and user's usage,
|
||||
adding primary key with integer type and involving createTime, updateTime, status fields should be somewhat meaningful.
|
||||
Of course, you could customize the schema per your concretely circumstance freely.
|
||||
|
||||
<pre><code>
|
||||
CREATE TABLE IF NOT EXISTS `filer_mapping` (
|
||||
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
`uriPath` char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
|
||||
`fid` char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
|
||||
`createTime` int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
|
||||
`updateTime` int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
|
||||
`remark` varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
|
||||
`status` tinyint(2) DEFAULT '1' COMMENT 'resource status',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `index_uriPath` (`uriPath`)
|
||||
) DEFAULT CHARSET=utf8;
|
||||
</code></pre>
|
||||
|
||||
|
||||
The MySQL 's config params is not added into the weed command option as other stores(redis,cassandra). Instead,
|
||||
We created a config file(json format) for them. TOML,YAML or XML also should be OK. But TOML and YAML need import thirdparty package
|
||||
while XML is a little bit complex.
|
||||
|
||||
The sample config file's content is below:
|
||||
|
||||
<pre><code>
|
||||
{
|
||||
"mysql": [
|
||||
{
|
||||
"User": "root",
|
||||
"Password": "root",
|
||||
"HostName": "127.0.0.1",
|
||||
"Port": 3306,
|
||||
"DataBase": "seaweedfs"
|
||||
},
|
||||
{
|
||||
"User": "root",
|
||||
"Password": "root",
|
||||
"HostName": "127.0.0.2",
|
||||
"Port": 3306,
|
||||
"DataBase": "seaweedfs"
|
||||
}
|
||||
],
|
||||
"IsSharding":true,
|
||||
"ShardCount":1024
|
||||
}
|
||||
</code></pre>
|
||||
|
||||
|
||||
The "mysql" field in above conf file is an array which include all mysql instances you prepared to store sharding data.
|
||||
|
||||
1. If one mysql instance is enough, just keep one instance in "mysql" field.
|
||||
|
||||
2. If table sharding at a specific mysql instance is needed , mark "IsSharding" field with true and specify total table sharding numbers using "ShardCount" field.
|
||||
|
||||
3. If the mysql service could be auto scaled transparently in your environment, just config one mysql instance(usually it's a frondend proxy or VIP),and mark "IsSharding" with false value
|
||||
|
||||
4. If you prepare more than one mysql instance and have no plan to use table sharding for any instance(mark isSharding with false), instance sharding will still be done implicitly
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,270 +0,0 @@
|
|||
package mysql_store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
const (
|
||||
sqlUrl = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
|
||||
default_maxIdleConnections = 100
|
||||
default_maxOpenConnections = 50
|
||||
default_maxTableNums = 1024
|
||||
tableName = "filer_mapping"
|
||||
)
|
||||
|
||||
var (
|
||||
_init_db sync.Once
|
||||
_db_connections []*sql.DB
|
||||
)
|
||||
|
||||
type MySqlConf struct {
|
||||
User string
|
||||
Password string
|
||||
HostName string
|
||||
Port int
|
||||
DataBase string
|
||||
MaxIdleConnections int
|
||||
MaxOpenConnections int
|
||||
}
|
||||
|
||||
type ShardingConf struct {
|
||||
IsSharding bool `json:"isSharding"`
|
||||
ShardCount int `json:"shardCount"`
|
||||
}
|
||||
|
||||
type MySqlStore struct {
|
||||
dbs []*sql.DB
|
||||
isSharding bool
|
||||
shardCount int
|
||||
}
|
||||
|
||||
func getDbConnection(confs []MySqlConf) []*sql.DB {
|
||||
_init_db.Do(func() {
|
||||
for _, conf := range confs {
|
||||
|
||||
sqlUrl := fmt.Sprintf(sqlUrl, conf.User, conf.Password, conf.HostName, conf.Port, conf.DataBase)
|
||||
var dbErr error
|
||||
_db_connection, dbErr := sql.Open("mysql", sqlUrl)
|
||||
if dbErr != nil {
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
panic(dbErr)
|
||||
}
|
||||
var maxIdleConnections, maxOpenConnections int
|
||||
|
||||
if conf.MaxIdleConnections != 0 {
|
||||
maxIdleConnections = conf.MaxIdleConnections
|
||||
} else {
|
||||
maxIdleConnections = default_maxIdleConnections
|
||||
}
|
||||
if conf.MaxOpenConnections != 0 {
|
||||
maxOpenConnections = conf.MaxOpenConnections
|
||||
} else {
|
||||
maxOpenConnections = default_maxOpenConnections
|
||||
}
|
||||
|
||||
_db_connection.SetMaxIdleConns(maxIdleConnections)
|
||||
_db_connection.SetMaxOpenConns(maxOpenConnections)
|
||||
_db_connections = append(_db_connections, _db_connection)
|
||||
}
|
||||
})
|
||||
return _db_connections
|
||||
}
|
||||
|
||||
func NewMysqlStore(confs []MySqlConf, isSharding bool, shardCount int) *MySqlStore {
|
||||
ms := &MySqlStore{
|
||||
dbs: getDbConnection(confs),
|
||||
isSharding: isSharding,
|
||||
shardCount: shardCount,
|
||||
}
|
||||
|
||||
for _, db := range ms.dbs {
|
||||
if !isSharding {
|
||||
ms.shardCount = 1
|
||||
} else {
|
||||
if ms.shardCount == 0 {
|
||||
ms.shardCount = default_maxTableNums
|
||||
}
|
||||
}
|
||||
for i := 0; i < ms.shardCount; i++ {
|
||||
if err := ms.createTables(db, tableName, i); err != nil {
|
||||
fmt.Printf("create table failed %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ms
|
||||
}
|
||||
|
||||
func (s *MySqlStore) hash(fullFileName string) (instance_offset, table_postfix int) {
|
||||
hash_value := crc32.ChecksumIEEE([]byte(fullFileName))
|
||||
instance_offset = int(hash_value) % len(s.dbs)
|
||||
table_postfix = int(hash_value) % s.shardCount
|
||||
return
|
||||
}
|
||||
|
||||
func (s *MySqlStore) parseFilerMappingInfo(path string) (instanceId int, tableFullName string, err error) {
|
||||
instance_offset, table_postfix := s.hash(path)
|
||||
instanceId = instance_offset
|
||||
if s.isSharding {
|
||||
tableFullName = fmt.Sprintf("%s_%04d", tableName, table_postfix)
|
||||
} else {
|
||||
tableFullName = tableName
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *MySqlStore) Get(fullFilePath string) (fid string, err error) {
|
||||
instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("MySqlStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
|
||||
}
|
||||
fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName)
|
||||
if err == sql.ErrNoRows {
|
||||
//Could not found
|
||||
err = filer.ErrNotFound
|
||||
}
|
||||
return fid, err
|
||||
}
|
||||
|
||||
func (s *MySqlStore) Put(fullFilePath string, fid string) (err error) {
|
||||
var tableFullName string
|
||||
|
||||
instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("MySqlStore Put operation can not parse file path %s: err is %v", fullFilePath, err)
|
||||
}
|
||||
var old_fid string
|
||||
if old_fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil && err != sql.ErrNoRows {
|
||||
return fmt.Errorf("MySqlStore Put operation failed when querying path %s: err is %v", fullFilePath, err)
|
||||
} else {
|
||||
if len(old_fid) == 0 {
|
||||
err = s.insert(fullFilePath, fid, s.dbs[instance_offset], tableFullName)
|
||||
err = fmt.Errorf("MySqlStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err)
|
||||
} else {
|
||||
err = s.update(fullFilePath, fid, s.dbs[instance_offset], tableFullName)
|
||||
err = fmt.Errorf("MySqlStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *MySqlStore) Delete(fullFilePath string) (err error) {
|
||||
var fid string
|
||||
instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("MySqlStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
|
||||
}
|
||||
if fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil {
|
||||
return fmt.Errorf("MySqlStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
|
||||
} else if fid == "" {
|
||||
return nil
|
||||
}
|
||||
if err = s.delete(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil {
|
||||
return fmt.Errorf("MySqlStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MySqlStore) Close() {
|
||||
for _, db := range s.dbs {
|
||||
db.Close()
|
||||
}
|
||||
}
|
||||
|
||||
var createTable = `
|
||||
CREATE TABLE IF NOT EXISTS %s (
|
||||
id bigint(20) NOT NULL AUTO_INCREMENT,
|
||||
uriPath char(255) NOT NULL DEFAULT "" COMMENT 'http uriPath',
|
||||
fid char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
|
||||
createTime int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
|
||||
updateTime int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
|
||||
remark varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
|
||||
status tinyint(2) DEFAULT '1' COMMENT 'resource status',
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY index_uriPath (uriPath)
|
||||
) DEFAULT CHARSET=utf8;
|
||||
`
|
||||
|
||||
func (s *MySqlStore) createTables(db *sql.DB, tableName string, postfix int) error {
|
||||
var realTableName string
|
||||
if s.isSharding {
|
||||
realTableName = fmt.Sprintf("%s_%4d", tableName, postfix)
|
||||
} else {
|
||||
realTableName = tableName
|
||||
}
|
||||
|
||||
stmt, err := db.Prepare(fmt.Sprintf(createTable, realTableName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
_, err = stmt.Exec()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MySqlStore) query(uriPath string, db *sql.DB, tableName string) (string, error) {
|
||||
sqlStatement := "SELECT fid FROM %s WHERE uriPath=?"
|
||||
row := db.QueryRow(fmt.Sprintf(sqlStatement, tableName), uriPath)
|
||||
var fid string
|
||||
err := row.Scan(&fid)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fid, nil
|
||||
}
|
||||
|
||||
func (s *MySqlStore) update(uriPath string, fid string, db *sql.DB, tableName string) error {
|
||||
sqlStatement := "UPDATE %s SET fid=?, updateTime=? WHERE uriPath=?"
|
||||
res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), fid, time.Now().Unix(), uriPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MySqlStore) insert(uriPath string, fid string, db *sql.DB, tableName string) error {
|
||||
sqlStatement := "INSERT INTO %s (uriPath,fid,createTime) VALUES(?,?,?)"
|
||||
res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath, fid, time.Now().Unix())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MySqlStore) delete(uriPath string, db *sql.DB, tableName string) error {
|
||||
sqlStatement := "DELETE FROM %s WHERE uriPath=?"
|
||||
res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
package mysql_store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"hash/crc32"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateMysqlConf(t *testing.T) {
|
||||
var conf []MySqlConf
|
||||
conf = append(conf, MySqlConf{
|
||||
User: "root",
|
||||
Password: "root",
|
||||
HostName: "localhost",
|
||||
Port: 3306,
|
||||
DataBase: "seaweedfs",
|
||||
})
|
||||
body, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
t.Errorf("json encoding err %s", err.Error())
|
||||
}
|
||||
t.Logf("json output is %s", string(body))
|
||||
}
|
||||
|
||||
func TestCRC32FullPathName(t *testing.T) {
|
||||
fullPathName := "/prod-bucket/law632191483895612493300-signed.pdf"
|
||||
hash_value := crc32.ChecksumIEEE([]byte(fullPathName))
|
||||
table_postfix := int(hash_value) % 1024
|
||||
t.Logf("table postfix %d", table_postfix)
|
||||
}
|
||||
|
|
@ -1,456 +0,0 @@
|
|||
package postgres_store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
_ "path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type DirectoryId int32
|
||||
|
||||
func databaseExists(db *sql.DB, databaseName string) (bool, error) {
|
||||
sqlStatement := "SELECT datname from pg_database WHERE datname='%s'"
|
||||
row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName))
|
||||
|
||||
var dbName string
|
||||
err := row.Scan(&dbName)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func createDatabase(db *sql.DB, databaseName string) error {
|
||||
sqlStatement := "CREATE DATABASE %s ENCODING='UTF8'"
|
||||
_, err := db.Exec(fmt.Sprintf(sqlStatement, databaseName))
|
||||
return err
|
||||
}
|
||||
|
||||
func getDbConnection(conf PostgresConf) *sql.DB {
|
||||
_init_db.Do(func() {
|
||||
|
||||
sqlUrl := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, "postgres", conf.SslMode)
|
||||
glog.V(3).Infoln("Opening postgres master database")
|
||||
|
||||
var dbErr error
|
||||
_db_connection, dbErr := sql.Open("postgres", sqlUrl)
|
||||
if dbErr != nil {
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
panic(dbErr)
|
||||
}
|
||||
|
||||
pingErr := _db_connection.Ping()
|
||||
if pingErr != nil {
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
panic(pingErr)
|
||||
}
|
||||
|
||||
glog.V(3).Infoln("Checking to see if DB exists: ", conf.DataBase)
|
||||
var existsErr error
|
||||
dbExists, existsErr := databaseExists(_db_connection, conf.DataBase)
|
||||
if existsErr != nil {
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
panic(existsErr)
|
||||
}
|
||||
|
||||
if !dbExists {
|
||||
glog.V(3).Infoln("Database doesn't exist. Attempting to create one: ", conf.DataBase)
|
||||
createErr := createDatabase(_db_connection, conf.DataBase)
|
||||
if createErr != nil {
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
panic(createErr)
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(3).Infoln("Closing master postgres database and opening configured database: ", conf.DataBase)
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
|
||||
sqlUrl = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, conf.DataBase, conf.SslMode)
|
||||
_db_connection, dbErr = sql.Open("postgres", sqlUrl)
|
||||
if dbErr != nil {
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
panic(dbErr)
|
||||
}
|
||||
|
||||
pingErr = _db_connection.Ping()
|
||||
if pingErr != nil {
|
||||
_db_connection.Close()
|
||||
_db_connection = nil
|
||||
panic(pingErr)
|
||||
}
|
||||
|
||||
maxIdleConnections, maxOpenConnections := default_maxIdleConnections, default_maxOpenConnections
|
||||
if conf.MaxIdleConnections != 0 {
|
||||
maxIdleConnections = conf.MaxIdleConnections
|
||||
}
|
||||
if conf.MaxOpenConnections != 0 {
|
||||
maxOpenConnections = conf.MaxOpenConnections
|
||||
}
|
||||
|
||||
_db_connection.SetMaxIdleConns(maxIdleConnections)
|
||||
_db_connection.SetMaxOpenConns(maxOpenConnections)
|
||||
})
|
||||
return _db_connection
|
||||
}
|
||||
|
||||
var createDirectoryTable = `
|
||||
|
||||
CREATE TABLE IF NOT EXISTS %s (
|
||||
id BIGSERIAL NOT NULL,
|
||||
directoryRoot VARCHAR(1024) NOT NULL DEFAULT '',
|
||||
directoryName VARCHAR(1024) NOT NULL DEFAULT '',
|
||||
CONSTRAINT unique_directory UNIQUE (directoryRoot, directoryName)
|
||||
);
|
||||
`
|
||||
|
||||
var createFileTable = `
|
||||
|
||||
CREATE TABLE IF NOT EXISTS %s (
|
||||
id BIGSERIAL NOT NULL,
|
||||
directoryPart VARCHAR(1024) NOT NULL DEFAULT '',
|
||||
filePart VARCHAR(1024) NOT NULL DEFAULT '',
|
||||
fid VARCHAR(36) NOT NULL DEFAULT '',
|
||||
createTime BIGINT NOT NULL DEFAULT 0,
|
||||
updateTime BIGINT NOT NULL DEFAULT 0,
|
||||
remark VARCHAR(20) NOT NULL DEFAULT '',
|
||||
status SMALLINT NOT NULL DEFAULT '1',
|
||||
PRIMARY KEY (id),
|
||||
CONSTRAINT %s_unique_file UNIQUE (directoryPart, filePart)
|
||||
);
|
||||
`
|
||||
|
||||
func (s *PostgresStore) createDirectoriesTable() error {
|
||||
glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", directoriesTableName)
|
||||
|
||||
sqlCreate := fmt.Sprintf(createDirectoryTable, directoriesTableName)
|
||||
|
||||
stmt, err := s.db.Prepare(sqlCreate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
_, err = stmt.Exec()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) createFilesTable() error {
|
||||
|
||||
glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", filesTableName)
|
||||
|
||||
sqlCreate := fmt.Sprintf(createFileTable, filesTableName, filesTableName)
|
||||
|
||||
stmt, err := s.db.Prepare(sqlCreate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
_, err = stmt.Exec()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) query(uriPath string) (string, error) {
|
||||
directoryPart, filePart := filepath.Split(uriPath)
|
||||
sqlStatement := fmt.Sprintf("SELECT fid FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
|
||||
|
||||
row := s.db.QueryRow(sqlStatement, directoryPart, filePart)
|
||||
var fid string
|
||||
err := row.Scan(&fid)
|
||||
|
||||
glog.V(3).Infof("Postgres query -- looking up path '%s' and found id '%s' ", uriPath, fid)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fid, nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) update(uriPath string, fid string) error {
|
||||
directoryPart, filePart := filepath.Split(uriPath)
|
||||
sqlStatement := fmt.Sprintf("UPDATE %s SET fid=$1, updateTime=$2 WHERE directoryPart=$3 AND filePart=$4", filesTableName)
|
||||
|
||||
glog.V(3).Infof("Postgres query -- updating path '%s' with id '%s'", uriPath, fid)
|
||||
|
||||
res, err := s.db.Exec(sqlStatement, fid, time.Now().Unix(), directoryPart, filePart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) insert(uriPath string, fid string) error {
|
||||
directoryPart, filePart := filepath.Split(uriPath)
|
||||
|
||||
existingId, _, _ := s.lookupDirectory(directoryPart)
|
||||
if existingId == 0 {
|
||||
s.recursiveInsertDirectory(directoryPart)
|
||||
}
|
||||
|
||||
sqlStatement := fmt.Sprintf("INSERT INTO %s (directoryPart,filePart,fid,createTime) VALUES($1, $2, $3, $4)", filesTableName)
|
||||
glog.V(3).Infof("Postgres query -- inserting path '%s' with id '%s'", uriPath, fid)
|
||||
|
||||
res, err := s.db.Exec(sqlStatement, directoryPart, filePart, fid, time.Now().Unix())
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := res.RowsAffected()
|
||||
if rows != 1 {
|
||||
return fmt.Errorf("Postgres insert -- rows affected = %d. Expecting 1", rows)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) recursiveInsertDirectory(dirPath string) {
|
||||
pathParts := strings.Split(dirPath, "/")
|
||||
|
||||
var workingPath string = "/"
|
||||
for _, part := range pathParts {
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
workingPath += (part + "/")
|
||||
existingId, _, _ := s.lookupDirectory(workingPath)
|
||||
if existingId == 0 {
|
||||
s.insertDirectory(workingPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresStore) insertDirectory(dirPath string) {
|
||||
pathParts := strings.Split(dirPath, "/")
|
||||
|
||||
directoryRoot := "/"
|
||||
directoryName := ""
|
||||
if len(pathParts) > 1 {
|
||||
directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
|
||||
directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
|
||||
} else if len(pathParts) == 1 {
|
||||
directoryRoot = "/"
|
||||
directoryName = pathParts[0] + "/"
|
||||
}
|
||||
sqlInsertDirectoryStatement := fmt.Sprintf("INSERT INTO %s (directoryroot, directoryname) "+
|
||||
"SELECT $1, $2 WHERE NOT EXISTS ( SELECT id FROM %s WHERE directoryroot=$3 AND directoryname=$4 )",
|
||||
directoriesTableName, directoriesTableName)
|
||||
|
||||
glog.V(4).Infof("Postgres query -- Inserting directory (if it doesn't exist) - root = %s, name = %s",
|
||||
directoryRoot, directoryName)
|
||||
|
||||
_, err := s.db.Exec(sqlInsertDirectoryStatement, directoryRoot, directoryName, directoryRoot, directoryName)
|
||||
if err != nil {
|
||||
glog.V(0).Infof("Postgres query -- Error inserting directory - root = %s, name = %s: %s",
|
||||
directoryRoot, directoryName, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresStore) delete(uriPath string) error {
|
||||
directoryPart, filePart := filepath.Split(uriPath)
|
||||
sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
|
||||
|
||||
glog.V(3).Infof("Postgres query -- deleting path '%s'", uriPath)
|
||||
|
||||
res, err := s.db.Exec(sqlStatement, directoryPart, filePart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) lookupDirectory(dirPath string) (DirectoryId, string, error) {
|
||||
directoryRoot, directoryName := s.mySplitPath(dirPath)
|
||||
|
||||
sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
|
||||
|
||||
row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName)
|
||||
var id DirectoryId
|
||||
var dirRoot string
|
||||
var dirName string
|
||||
err := row.Scan(&id, &dirRoot, &dirName)
|
||||
|
||||
glog.V(3).Infof("Postgres lookupDirectory -- looking up directory '%s' and found id '%d', root '%s', name '%s' ", dirPath, id, dirRoot, dirName)
|
||||
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return id, filepath.Join(dirRoot, dirName), err
|
||||
}
|
||||
|
||||
func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryName, err error) {
|
||||
sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName)
|
||||
rows, err := s.db.Query(sqlStatement, dirPath, limit)
|
||||
|
||||
if err != nil {
|
||||
glog.V(0).Infof("Postgres findDirectories error: %s", err)
|
||||
}
|
||||
|
||||
if rows != nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id DirectoryId
|
||||
var directoryRoot string
|
||||
var directoryName string
|
||||
|
||||
scanErr := rows.Scan(&id, &directoryRoot, &directoryName)
|
||||
if scanErr != nil {
|
||||
err = scanErr
|
||||
}
|
||||
dirs = append(dirs, filer.DirectoryName(directoryName))
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bool {
|
||||
if recursive {
|
||||
return true
|
||||
}
|
||||
sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName)
|
||||
row := s.db.QueryRow(sqlStatement, dirPath+"%")
|
||||
|
||||
var id DirectoryId
|
||||
err := row.Scan(&id)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *PostgresStore) mySplitPath(dirPath string) (directoryRoot string, directoryName string) {
|
||||
pathParts := strings.Split(dirPath, "/")
|
||||
directoryRoot = "/"
|
||||
directoryName = ""
|
||||
if len(pathParts) > 1 {
|
||||
directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
|
||||
directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
|
||||
} else if len(pathParts) == 1 {
|
||||
directoryRoot = "/"
|
||||
directoryName = pathParts[0] + "/"
|
||||
}
|
||||
return directoryRoot, directoryName
|
||||
}
|
||||
|
||||
func (s *PostgresStore) deleteDirectory(dirPath string, recursive bool) (err error) {
|
||||
directoryRoot, directoryName := s.mySplitPath(dirPath)
|
||||
|
||||
// delete files
|
||||
sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directorypart=$1", filesTableName)
|
||||
_, err = s.db.Exec(sqlStatement, dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete specific directory if it is empty or recursive delete was requested
|
||||
safeToDelete := s.safeToDeleteDirectory(dirPath, recursive)
|
||||
if safeToDelete {
|
||||
sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
|
||||
_, err = s.db.Exec(sqlStatement, directoryRoot, directoryName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if recursive {
|
||||
// delete descendant files
|
||||
sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directorypart LIKE $1", filesTableName)
|
||||
_, err = s.db.Exec(sqlStatement, dirPath+"%")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete descendant directories
|
||||
sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot LIKE $1", directoriesTableName)
|
||||
_, err = s.db.Exec(sqlStatement, dirPath+"%")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *PostgresStore) findFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
|
||||
var rows *sql.Rows = nil
|
||||
|
||||
if lastFileName == "" {
|
||||
sqlStatement :=
|
||||
fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 ORDER BY id LIMIT $2", filesTableName)
|
||||
rows, err = s.db.Query(sqlStatement, dirPath, limit)
|
||||
} else {
|
||||
sqlStatement :=
|
||||
fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 "+
|
||||
"AND id > (SELECT id FROM %s WHERE directoryPart=$2 AND filepart=$3) ORDER BY id LIMIT $4",
|
||||
filesTableName, filesTableName)
|
||||
_, lastFileNameName := filepath.Split(lastFileName)
|
||||
rows, err = s.db.Query(sqlStatement, dirPath, dirPath, lastFileNameName, limit)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
glog.V(0).Infof("Postgres find files error: %s", err)
|
||||
}
|
||||
|
||||
if rows != nil {
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var fid filer.FileId
|
||||
var directoryPart string
|
||||
var filePart string
|
||||
|
||||
scanErr := rows.Scan(&fid, &directoryPart, &filePart)
|
||||
if scanErr != nil {
|
||||
err = scanErr
|
||||
}
|
||||
|
||||
files = append(files, filer.FileEntry{Name: filepath.Join(directoryPart, filePart), Id: fid})
|
||||
if len(files) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(3).Infof("Postgres findFiles -- looking up files under '%s' and found %d files. Limit=%d, lastFileName=%s",
|
||||
dirPath, len(files), limit, lastFileName)
|
||||
|
||||
return files, err
|
||||
}
|
||||
|
|
@ -1,149 +0,0 @@
|
|||
package postgres_store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
_ "path/filepath"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const (
|
||||
default_maxIdleConnections = 100
|
||||
default_maxOpenConnections = 50
|
||||
filesTableName = "files"
|
||||
directoriesTableName = "directories"
|
||||
)
|
||||
|
||||
var (
|
||||
_init_db sync.Once
|
||||
_db_connection *sql.DB
|
||||
)
|
||||
|
||||
type PostgresConf struct {
|
||||
User string
|
||||
Password string
|
||||
HostName string
|
||||
Port int
|
||||
DataBase string
|
||||
SslMode string
|
||||
MaxIdleConnections int
|
||||
MaxOpenConnections int
|
||||
}
|
||||
|
||||
type PostgresStore struct {
|
||||
db *sql.DB
|
||||
server string
|
||||
user string
|
||||
password string
|
||||
}
|
||||
|
||||
func (s *PostgresStore) CreateFile(fullFilePath string, fid string) (err error) {
|
||||
|
||||
var old_fid string
|
||||
if old_fid, err = s.query(fullFilePath); err != nil && err != sql.ErrNoRows {
|
||||
return fmt.Errorf("PostgresStore Put operation failed when querying path %s: err is %v", fullFilePath, err)
|
||||
} else {
|
||||
if len(old_fid) == 0 {
|
||||
err = s.insert(fullFilePath, fid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PostgresStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err)
|
||||
}
|
||||
} else {
|
||||
err = s.update(fullFilePath, fid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PostgresStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func (s *PostgresStore) FindFile(fullFilePath string) (fid string, err error) {
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("PostgresStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
|
||||
}
|
||||
fid, err = s.query(fullFilePath)
|
||||
|
||||
return fid, err
|
||||
}
|
||||
|
||||
func (s *PostgresStore) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) {
|
||||
fullPath := filepath.Join(dirPath, name)
|
||||
if fileId, err = s.FindFile(fullPath); err == nil {
|
||||
return true, fileId, nil
|
||||
}
|
||||
if _, _, err := s.lookupDirectory(fullPath); err == nil {
|
||||
return true, "", err
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
|
||||
func (s *PostgresStore) DeleteFile(fullFilePath string) (fid string, err error) {
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
|
||||
}
|
||||
if fid, err = s.query(fullFilePath); err != nil {
|
||||
return "", fmt.Errorf("PostgresStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
|
||||
} else if fid == "" {
|
||||
return "", nil
|
||||
}
|
||||
if err = s.delete(fullFilePath); err != nil {
|
||||
return "", fmt.Errorf("PostgresStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
|
||||
} else {
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) {
|
||||
|
||||
dirs, err = s.findDirectories(dirPath, 1000)
|
||||
|
||||
glog.V(3).Infof("Postgres ListDirs = found %d directories under %s", len(dirs), dirPath)
|
||||
|
||||
return dirs, err
|
||||
}
|
||||
|
||||
func (s *PostgresStore) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
|
||||
files, err = s.findFiles(dirPath, lastFileName, limit)
|
||||
return files, err
|
||||
}
|
||||
|
||||
func (s *PostgresStore) DeleteDirectory(dirPath string, recursive bool) (err error) {
|
||||
err = s.deleteDirectory(dirPath, recursive)
|
||||
if err != nil {
|
||||
glog.V(0).Infof("Error in Postgres DeleteDir '%s' (recursive = '%t'): %s", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *PostgresStore) Move(fromPath string, toPath string) (err error) {
|
||||
glog.V(3).Infoln("Calling posgres_store Move")
|
||||
return errors.New("Move is not yet implemented for the PostgreSQL store.")
|
||||
}
|
||||
|
||||
//func NewPostgresStore(master string, confs []PostgresConf, isSharding bool, shardCount int) *PostgresStore {
|
||||
func NewPostgresStore(master string, conf PostgresConf) *PostgresStore {
|
||||
pg := &PostgresStore{
|
||||
db: getDbConnection(conf),
|
||||
}
|
||||
|
||||
pg.createDirectoriesTable()
|
||||
|
||||
if err := pg.createFilesTable(); err != nil {
|
||||
fmt.Printf("create table failed %v", err)
|
||||
}
|
||||
|
||||
return pg
|
||||
}
|
||||
|
||||
func (s *PostgresStore) Close() {
|
||||
s.db.Close()
|
||||
}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
package redis_store
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
|
||||
"github.com/go-redis/redis"
|
||||
)
|
||||
|
||||
type RedisStore struct {
|
||||
Client *redis.Client
|
||||
}
|
||||
|
||||
func NewRedisStore(hostPort string, password string, database int) *RedisStore {
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: hostPort,
|
||||
Password: password,
|
||||
DB: database,
|
||||
})
|
||||
return &RedisStore{Client: client}
|
||||
}
|
||||
|
||||
func (s *RedisStore) Get(fullFileName string) (fid string, err error) {
|
||||
fid, err = s.Client.Get(fullFileName).Result()
|
||||
if err == redis.Nil {
|
||||
err = filer.ErrNotFound
|
||||
}
|
||||
return fid, err
|
||||
}
|
||||
func (s *RedisStore) Put(fullFileName string, fid string) (err error) {
|
||||
_, err = s.Client.Set(fullFileName, fid, 0).Result()
|
||||
if err == redis.Nil {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Currently the fid is not returned
|
||||
func (s *RedisStore) Delete(fullFileName string) (err error) {
|
||||
_, err = s.Client.Del(fullFileName).Result()
|
||||
if err == redis.Nil {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *RedisStore) Close() {
|
||||
if s.Client != nil {
|
||||
s.Client.Close()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
There are two main components of a filer: directories and files.
|
||||
|
||||
My previous approach was to use some sequance number to generate directoryId.
|
||||
However, this is not scalable. The id generation itself is a bottleneck.
|
||||
It needs careful locking and deduplication checking to get a directoryId.
|
||||
|
||||
In a second design, each directory is deterministically mapped to UUID version 3,
|
||||
which uses MD5 to map a tuple of <uuid, name> to a version 3 UUID.
|
||||
However, this UUID3 approach is logically the same as storing the full path.
|
||||
|
||||
Storing the full path is the simplest design.
|
||||
|
||||
separator is a special byte, 0x00.
|
||||
|
||||
When writing a file:
|
||||
<file parent full path, separator, file name> => fildId, file properties
|
||||
For folders:
|
||||
The filer breaks the directory path into folders.
|
||||
for each folder:
|
||||
if it is not in cache:
|
||||
check whether the folder is created in the KVS, if not:
|
||||
set <folder parent full path, separator, folder name> => directory properties
|
||||
if no permission for the folder:
|
||||
break
|
||||
|
||||
|
||||
The filer caches the most recently used folder permissions with a TTL.
|
||||
So any folder permission change needs to wait TTL interval to take effect.
|
||||
|
||||
|
||||
|
||||
When listing the directory:
|
||||
prefix scan of using (the folder full path + separator) as the prefix
|
||||
|
||||
The downside:
|
||||
1. Rename a folder will need to recursively process all sub folders and files.
|
||||
2. Move a folder will need to recursively process all sub folders and files.
|
||||
So these operations are not allowed if the folder is not empty.
|
||||
|
||||
Allowing:
|
||||
1. Rename a file
|
||||
2. Move a file to a different folder
|
||||
3. Delete an empty folder
|
||||
|
||||
|
||||
|
|
@ -319,7 +319,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
|
|||
if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil {
|
||||
oldFid := entry.Chunks[0].FileId
|
||||
operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid))
|
||||
} else if err != nil && err != filer.ErrNotFound {
|
||||
} else if err != nil && err != filer2.ErrNotFound {
|
||||
glog.V(0).Infof("error %v occur when finding %s in filer store", err, path)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue