This commit is contained in:
Chris Lu 2018-05-27 11:52:26 -07:00
parent 8647191bee
commit 458ada173e
36 changed files with 105 additions and 107 deletions

View file

@ -5,14 +5,14 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"github.com/soheilhy/cmux" "github.com/soheilhy/cmux"
"google.golang.org/grpc/reflection"
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/filer2"
) )
var ( var (

View file

@ -8,9 +8,9 @@ import (
"bazil.org/fuse" "bazil.org/fuse"
"bazil.org/fuse/fs" "bazil.org/fuse/fs"
"github.com/chrislusf/seaweedfs/weed/filesys"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filesys"
) )
func runMount(cmd *Command, args []string) bool { func runMount(cmd *Command, args []string) bool {

View file

@ -1,8 +1,8 @@
package abstract_sql package abstract_sql
import ( import (
"fmt"
"database/sql" "database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"

View file

@ -2,9 +2,9 @@ package cassandra
import ( import (
"fmt" "fmt"
"github.com/gocql/gocql"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/gocql/gocql"
"github.com/spf13/viper" "github.com/spf13/viper"
) )

View file

@ -3,8 +3,8 @@ package filer2
import ( import (
"os" "os"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
) )
const ( const (
@ -20,12 +20,11 @@ enabled = false
enabled = false enabled = false
dir = "." # directory to store level db files dir = "." # directory to store level db files
[mysql] ####################################################
# multiple filers on shared storage, fairly scalable # multiple filers on shared storage, fairly scalable
# ####################################################
# need to choose or create a database.
# need to manually create a table "filemeta". [mysql]
#
# CREATE TABLE IF NOT EXISTS filemeta ( # CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) COMMENT 'directory or file name', # name VARCHAR(1000) COMMENT 'directory or file name',
@ -33,7 +32,6 @@ dir = "." # directory to store level db files
# meta BLOB, # meta BLOB,
# PRIMARY KEY (dirhash, name) # PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8; # ) DEFAULT CHARSET=utf8;
#
enabled = true enabled = true
hostname = "localhost" hostname = "localhost"
port = 3306 port = 3306

View file

@ -15,7 +15,7 @@ type Attr struct {
Gid uint32 // group gid Gid uint32 // group gid
} }
func (attr Attr) IsDirectory() (bool) { func (attr Attr) IsDirectory() bool {
return attr.Mode&os.ModeDir > 0 return attr.Mode&os.ModeDir > 0
} }

View file

@ -4,9 +4,9 @@ import (
"os" "os"
"time" "time"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/proto"
"fmt"
) )
func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) { func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
@ -23,7 +23,7 @@ func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
return proto.Marshal(message) return proto.Marshal(message)
} }
func (entry *Entry) DecodeAttributesAndChunks(blob []byte) (error) { func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
message := &filer_pb.Entry{} message := &filer_pb.Entry{}

View file

@ -1,9 +1,9 @@
package filer2 package filer2
import ( import (
"sort"
"log" "log"
"math" "math"
"sort"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
) )

View file

@ -1,8 +1,8 @@
package filer2 package filer2
import ( import (
"testing"
"log" "log"
"testing"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
) )

View file

@ -3,12 +3,12 @@ package filer2
import ( import (
"fmt" "fmt"
"github.com/karlseguin/ccache"
"strings"
"path/filepath"
"time"
"os"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/karlseguin/ccache"
"os"
"path/filepath"
"strings"
"time"
) )
type Filer struct { type Filer struct {
@ -24,15 +24,15 @@ func NewFiler(master string) *Filer {
} }
} }
func (f *Filer) SetStore(store FilerStore) () { func (f *Filer) SetStore(store FilerStore) {
f.store = store f.store = store
} }
func (f *Filer) DisableDirectoryCache() () { func (f *Filer) DisableDirectoryCache() {
f.directoryCache = nil f.directoryCache = nil
} }
func (f *Filer) CreateEntry(entry *Entry) (error) { func (f *Filer) CreateEntry(entry *Entry) error {
dirParts := strings.Split(string(entry.FullPath), "/") dirParts := strings.Split(string(entry.FullPath), "/")
@ -140,7 +140,7 @@ func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive
return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit) return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
} }
func (f *Filer) cacheGetDirectory(dirpath string) (*Entry) { func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
if f.directoryCache == nil { if f.directoryCache == nil {
return nil return nil
} }

View file

@ -7,8 +7,8 @@ import (
type FilerStore interface { type FilerStore interface {
GetName() string GetName() string
Initialize(viper *viper.Viper) (error) Initialize(viper *viper.Viper) error
InsertEntry(*Entry) (error) InsertEntry(*Entry) error
UpdateEntry(*Entry) (err error) UpdateEntry(*Entry) (err error)
FindEntry(FullPath) (entry *Entry, err error) FindEntry(FullPath) (entry *Entry, err error)
DeleteEntry(FullPath) (fileEntry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error)

View file

@ -25,7 +25,7 @@ func (fp FullPath) DirAndName() (string, string) {
return dir[:len(dir)-1], name return dir[:len(dir)-1], name
} }
func (fp FullPath) Name() (string) { func (fp FullPath) Name() string {
_, name := filepath.Split(string(fp)) _, name := filepath.Split(string(fp))
return name return name
} }

View file

@ -1,15 +1,15 @@
package leveldb package leveldb
import ( import (
"fmt"
"bytes" "bytes"
"fmt"
"github.com/syndtr/goleveldb/leveldb"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
weed_util "github.com/chrislusf/seaweedfs/weed/util" weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
"github.com/syndtr/goleveldb/leveldb"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
) )
const ( const (
@ -162,7 +162,7 @@ func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyP
return keyPrefix return keyPrefix
} }
func getNameFromKey(key []byte) (string) { func getNameFromKey(key []byte) string {
sepIndex := len(key) - 1 sepIndex := len(key) - 1
for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR { for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR {

View file

@ -1,10 +1,10 @@
package leveldb package leveldb
import ( import (
"testing"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"io/ioutil" "io/ioutil"
"os" "os"
"testing"
) )
func TestCreateAndFind(t *testing.T) { func TestCreateAndFind(t *testing.T) {

View file

@ -1,11 +1,11 @@
package memdb package memdb
import ( import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/google/btree" "github.com/google/btree"
"strings"
"fmt"
"github.com/spf13/viper" "github.com/spf13/viper"
"strings"
) )
func init() { func init() {

View file

@ -1,8 +1,8 @@
package memdb package memdb
import ( import (
"testing"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"testing"
) )
func TestCreateAndFind(t *testing.T) { func TestCreateAndFind(t *testing.T) {

View file

@ -1,13 +1,13 @@
package mysql package mysql
import ( import (
"fmt"
"database/sql" "database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/spf13/viper"
_ "github.com/go-sql-driver/mysql"
"github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql"
_ "github.com/go-sql-driver/mysql"
"github.com/spf13/viper"
) )
const ( const (

View file

@ -1,13 +1,13 @@
package postgres package postgres
import ( import (
"fmt"
"database/sql" "database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/spf13/viper"
_ "github.com/lib/pq"
"github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql"
_ "github.com/lib/pq"
"github.com/spf13/viper"
) )
const ( const (

View file

@ -4,8 +4,8 @@ import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
"github.com/go-redis/redis" "github.com/go-redis/redis"
"github.com/spf13/viper"
"sort" "sort"
"strings" "strings"
) )

View file

@ -7,12 +7,12 @@ import (
"path" "path"
"sync" "sync"
"bazil.org/fuse/fs"
"bazil.org/fuse" "bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"time"
"path/filepath" "path/filepath"
"time"
) )
type Dir struct { type Dir struct {

View file

@ -1,15 +1,15 @@
package filesys package filesys
import ( import (
"context"
"bazil.org/fuse" "bazil.org/fuse"
"bazil.org/fuse/fs" "bazil.org/fuse/fs"
"context"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"path/filepath"
"os" "os"
"path/filepath"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/filer2"
) )
var _ = fs.Node(&File{}) var _ = fs.Node(&File{})

View file

@ -1,19 +1,19 @@
package filesys package filesys
import ( import (
"bazil.org/fuse/fs"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/filer2"
"context"
"github.com/chrislusf/seaweedfs/weed/glog"
"bazil.org/fuse" "bazil.org/fuse"
"bazil.org/fuse/fs"
"bytes" "bytes"
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"time" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"strings" "strings"
"sync" "sync"
"github.com/chrislusf/seaweedfs/weed/util" "time"
) )
type FileHandle struct { type FileHandle struct {
@ -32,6 +32,7 @@ type FileHandle struct {
} }
var _ = fs.Handle(&FileHandle{}) var _ = fs.Handle(&FileHandle{})
// var _ = fs.HandleReadAller(&FileHandle{}) // var _ = fs.HandleReadAller(&FileHandle{})
var _ = fs.HandleReader(&FileHandle{}) var _ = fs.HandleReader(&FileHandle{})
var _ = fs.HandleFlusher(&FileHandle{}) var _ = fs.HandleFlusher(&FileHandle{})

View file

@ -3,9 +3,9 @@ package filesys
import ( import (
"bazil.org/fuse/fs" "bazil.org/fuse/fs"
"fmt" "fmt"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/karlseguin/ccache" "github.com/karlseguin/ccache"
"google.golang.org/grpc"
) )
type WFS struct { type WFS struct {

View file

@ -181,6 +181,7 @@ type bintree struct {
Func func() (*asset, error) Func func() (*asset, error)
Children map[string]*bintree Children map[string]*bintree
} }
var _bintree = &bintree{nil, map[string]*bintree{ var _bintree = &bintree{nil, map[string]*bintree{
"favicon": &bintree{nil, map[string]*bintree{ "favicon": &bintree{nil, map[string]*bintree{
"favicon.ico": &bintree{favicon, map[string]*bintree{}}, "favicon.ico": &bintree{favicon, map[string]*bintree{}},
@ -233,4 +234,3 @@ func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1) cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
} }

View file

@ -1,16 +1,16 @@
package weed_server package weed_server
import ( import (
"fmt"
"context" "context"
"time" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"time"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
) )
func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {

View file

@ -7,10 +7,6 @@ import (
"sync" "sync"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
_ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra"
_ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
@ -18,6 +14,10 @@ import (
_ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql"
_ "github.com/chrislusf/seaweedfs/weed/filer2/postgres" _ "github.com/chrislusf/seaweedfs/weed/filer2/postgres"
_ "github.com/chrislusf/seaweedfs/weed/filer2/redis" _ "github.com/chrislusf/seaweedfs/weed/filer2/redis"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
) )
type FilerServer struct { type FilerServer struct {

View file

@ -3,10 +3,10 @@ package weed_server
import ( import (
"net/http" "net/http"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"strconv" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"strconv"
"time" "time"
) )

View file

@ -7,11 +7,11 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filer2"
) )
// listDirectoryHandler lists directories and folers under a directory // listDirectoryHandler lists directories and folers under a directory

View file

@ -17,13 +17,12 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"time" "time"
) )

View file

@ -1,8 +1,8 @@
package master_ui package master_ui
import ( import (
"strings"
"path/filepath" "path/filepath"
"strings"
) )
type Breadcrumb struct { type Breadcrumb struct {

View file

@ -8,8 +8,8 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
) )
func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -5,9 +5,9 @@ import (
"path/filepath" "path/filepath"
"time" "time"
ui "github.com/chrislusf/seaweedfs/weed/server/volume_server_ui"
"github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
ui "github.com/chrislusf/seaweedfs/weed/server/volume_server_ui"
) )
func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -12,7 +12,7 @@ func getActualSize(size uint32) int64 {
return NeedleHeaderSize + int64(size) + NeedleChecksumSize + int64(padding) return NeedleHeaderSize + int64(size) + NeedleChecksumSize + int64(padding)
} }
func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (error) { func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error {
var indexSize int64 var indexSize int64
var e error var e error
if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil { if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil {