This commit is contained in:
Chris Lu 2018-05-27 11:52:26 -07:00
parent 8647191bee
commit 458ada173e
36 changed files with 105 additions and 107 deletions

View file

@ -5,14 +5,14 @@ import (
"strconv"
"time"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/soheilhy/cmux"
"google.golang.org/grpc/reflection"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/filer2"
"google.golang.org/grpc/reflection"
)
var (

View file

@ -68,7 +68,7 @@ func runCopy(cmd *Command, args []string) bool {
return false
}
filerDestination := args[len(args)-1]
fileOrDirs := args[0: len(args)-1]
fileOrDirs := args[0 : len(args)-1]
filerUrl, err := url.Parse(filerDestination)
if err != nil {

View file

@ -8,9 +8,9 @@ import (
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/chrislusf/seaweedfs/weed/filesys"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filesys"
)
func runMount(cmd *Command, args []string) bool {

View file

@ -1,8 +1,8 @@
package abstract_sql
import (
"fmt"
"database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"

View file

@ -2,9 +2,9 @@ package cassandra
import (
"fmt"
"github.com/gocql/gocql"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/gocql/gocql"
"github.com/spf13/viper"
)

View file

@ -3,8 +3,8 @@ package filer2
import (
"os"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
)
const (
@ -20,12 +20,11 @@ enabled = false
enabled = false
dir = "." # directory to store level db files
[mysql]
####################################################
# multiple filers on shared storage, fairly scalable
#
# need to choose or create a database.
# need to manually create a table "filemeta".
#
####################################################
[mysql]
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) COMMENT 'directory or file name',
@ -33,7 +32,6 @@ dir = "." # directory to store level db files
# meta BLOB,
# PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8;
#
enabled = true
hostname = "localhost"
port = 3306
@ -90,10 +88,10 @@ var (
func (f *Filer) LoadConfiguration() {
// find a filer store
viper.SetConfigName("filer") // name of config file (without extension)
viper.AddConfigPath(".") // optionally look for config in the working directory
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
viper.SetConfigName("filer") // name of config file (without extension)
viper.AddConfigPath(".") // optionally look for config in the working directory
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
if err := viper.ReadInConfig(); err != nil { // Handle errors reading the config file
glog.Fatalf("Failed to load filer.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/" +
"\n\nPlease follow this example and add a filer.toml file to " +

View file

@ -15,7 +15,7 @@ type Attr struct {
Gid uint32 // group gid
}
func (attr Attr) IsDirectory() (bool) {
func (attr Attr) IsDirectory() bool {
return attr.Mode&os.ModeDir > 0
}

View file

@ -4,9 +4,9 @@ import (
"os"
"time"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/gogo/protobuf/proto"
"fmt"
)
func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
@ -23,7 +23,7 @@ func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
return proto.Marshal(message)
}
func (entry *Entry) DecodeAttributesAndChunks(blob []byte) (error) {
func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
message := &filer_pb.Entry{}

View file

@ -1,9 +1,9 @@
package filer2
import (
"sort"
"log"
"math"
"sort"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

View file

@ -1,8 +1,8 @@
package filer2
import (
"testing"
"log"
"testing"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
@ -163,9 +163,9 @@ func TestChunksReading(t *testing.T) {
Offset: 0,
Size: 250,
Expected: []*ChunkView{
{Offset: 0, Size: 100, FileId: "abc", LogicOffset:0},
{Offset: 0, Size: 100, FileId: "asdf", LogicOffset:100},
{Offset: 0, Size: 50, FileId: "fsad", LogicOffset:200},
{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
{Offset: 0, Size: 50, FileId: "fsad", LogicOffset: 200},
},
},
// case 1: updates overwrite full chunks
@ -177,7 +177,7 @@ func TestChunksReading(t *testing.T) {
Offset: 50,
Size: 100,
Expected: []*ChunkView{
{Offset: 50, Size: 100, FileId: "asdf", LogicOffset:50},
{Offset: 50, Size: 100, FileId: "asdf", LogicOffset: 50},
},
},
// case 2: updates overwrite part of previous chunks
@ -189,8 +189,8 @@ func TestChunksReading(t *testing.T) {
Offset: 25,
Size: 50,
Expected: []*ChunkView{
{Offset: 25, Size: 25, FileId: "asdf", LogicOffset:25},
{Offset: 0, Size: 25, FileId: "abc", LogicOffset:50},
{Offset: 25, Size: 25, FileId: "asdf", LogicOffset: 25},
{Offset: 0, Size: 25, FileId: "abc", LogicOffset: 50},
},
},
// case 3: updates overwrite full chunks
@ -203,8 +203,8 @@ func TestChunksReading(t *testing.T) {
Offset: 0,
Size: 200,
Expected: []*ChunkView{
{Offset: 0, Size: 50, FileId: "asdf", LogicOffset:0},
{Offset: 0, Size: 150, FileId: "xxxx", LogicOffset:50},
{Offset: 0, Size: 50, FileId: "asdf", LogicOffset: 0},
{Offset: 0, Size: 150, FileId: "xxxx", LogicOffset: 50},
},
},
// case 4: updates far away from prev chunks
@ -217,7 +217,7 @@ func TestChunksReading(t *testing.T) {
Offset: 0,
Size: 400,
Expected: []*ChunkView{
{Offset: 0, Size: 200, FileId: "asdf", LogicOffset:0},
{Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0},
// {Offset: 0, Size: 150, FileId: "xxxx"}, // missing intervals should not happen
},
},
@ -232,8 +232,8 @@ func TestChunksReading(t *testing.T) {
Offset: 0,
Size: 220,
Expected: []*ChunkView{
{Offset: 0, Size: 200, FileId: "asdf", LogicOffset:0},
{Offset: 0, Size: 20, FileId: "abc", LogicOffset:200},
{Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0},
{Offset: 0, Size: 20, FileId: "abc", LogicOffset: 200},
},
},
// case 6: same updates
@ -246,7 +246,7 @@ func TestChunksReading(t *testing.T) {
Offset: 0,
Size: 100,
Expected: []*ChunkView{
{Offset: 0, Size: 100, FileId: "abc", LogicOffset:0},
{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
},
},
// case 7: edge cases
@ -259,8 +259,8 @@ func TestChunksReading(t *testing.T) {
Offset: 0,
Size: 200,
Expected: []*ChunkView{
{Offset: 0, Size: 100, FileId: "abc", LogicOffset:0},
{Offset: 0, Size: 100, FileId: "asdf", LogicOffset:100},
{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
},
},
}

View file

@ -3,12 +3,12 @@ package filer2
import (
"fmt"
"github.com/karlseguin/ccache"
"strings"
"path/filepath"
"time"
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/karlseguin/ccache"
"os"
"path/filepath"
"strings"
"time"
)
type Filer struct {
@ -24,15 +24,15 @@ func NewFiler(master string) *Filer {
}
}
func (f *Filer) SetStore(store FilerStore) () {
func (f *Filer) SetStore(store FilerStore) {
f.store = store
}
func (f *Filer) DisableDirectoryCache() () {
func (f *Filer) DisableDirectoryCache() {
f.directoryCache = nil
}
func (f *Filer) CreateEntry(entry *Entry) (error) {
func (f *Filer) CreateEntry(entry *Entry) error {
dirParts := strings.Split(string(entry.FullPath), "/")
@ -94,11 +94,11 @@ func (f *Filer) CreateEntry(entry *Entry) (error) {
}
/*
if !hasWritePermission(lastDirectoryEntry, entry) {
glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
}
if !hasWritePermission(lastDirectoryEntry, entry) {
glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
}
*/
if err := f.store.InsertEntry(entry); err != nil {
@ -135,12 +135,12 @@ func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) {
func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
if strings.HasSuffix(string(p), "/") && len(p) > 1 {
p = p[0:len(p)-1]
p = p[0 : len(p)-1]
}
return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
}
func (f *Filer) cacheGetDirectory(dirpath string) (*Entry) {
func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
if f.directoryCache == nil {
return nil
}

View file

@ -7,8 +7,8 @@ import (
type FilerStore interface {
GetName() string
Initialize(viper *viper.Viper) (error)
InsertEntry(*Entry) (error)
Initialize(viper *viper.Viper) error
InsertEntry(*Entry) error
UpdateEntry(*Entry) (err error)
FindEntry(FullPath) (entry *Entry, err error)
DeleteEntry(FullPath) (fileEntry *Entry, err error)

View file

@ -25,7 +25,7 @@ func (fp FullPath) DirAndName() (string, string) {
return dir[:len(dir)-1], name
}
func (fp FullPath) Name() (string) {
func (fp FullPath) Name() string {
_, name := filepath.Split(string(fp))
return name
}

View file

@ -1,15 +1,15 @@
package leveldb
import (
"fmt"
"bytes"
"fmt"
"github.com/syndtr/goleveldb/leveldb"
"github.com/chrislusf/seaweedfs/weed/filer2"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/spf13/viper"
"github.com/syndtr/goleveldb/leveldb"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
)
const (
@ -162,7 +162,7 @@ func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyP
return keyPrefix
}
func getNameFromKey(key []byte) (string) {
func getNameFromKey(key []byte) string {
sepIndex := len(key) - 1
for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR {

View file

@ -1,10 +1,10 @@
package leveldb
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/filer2"
"io/ioutil"
"os"
"testing"
)
func TestCreateAndFind(t *testing.T) {

View file

@ -1,11 +1,11 @@
package memdb
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/google/btree"
"strings"
"fmt"
"github.com/spf13/viper"
"strings"
)
func init() {

View file

@ -1,8 +1,8 @@
package memdb
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/filer2"
"testing"
)
func TestCreateAndFind(t *testing.T) {

View file

@ -1,13 +1,13 @@
package mysql
import (
"fmt"
"database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/spf13/viper"
_ "github.com/go-sql-driver/mysql"
"github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql"
_ "github.com/go-sql-driver/mysql"
"github.com/spf13/viper"
)
const (

View file

@ -1,13 +1,13 @@
package postgres
import (
"fmt"
"database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/spf13/viper"
_ "github.com/lib/pq"
"github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql"
_ "github.com/lib/pq"
"github.com/spf13/viper"
)
const (

View file

@ -4,8 +4,8 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
"github.com/go-redis/redis"
"github.com/spf13/viper"
"sort"
"strings"
)

View file

@ -7,12 +7,12 @@ import (
"path"
"sync"
"bazil.org/fuse/fs"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"time"
"path/filepath"
"time"
)
type Dir struct {

View file

@ -1,15 +1,15 @@
package filesys
import (
"context"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"context"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"path/filepath"
"os"
"path/filepath"
"time"
"github.com/chrislusf/seaweedfs/weed/filer2"
)
var _ = fs.Node(&File{})

View file

@ -1,19 +1,19 @@
package filesys
import (
"bazil.org/fuse/fs"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/filer2"
"context"
"github.com/chrislusf/seaweedfs/weed/glog"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"bytes"
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"strings"
"sync"
"github.com/chrislusf/seaweedfs/weed/util"
"time"
)
type FileHandle struct {
@ -32,6 +32,7 @@ type FileHandle struct {
}
var _ = fs.Handle(&FileHandle{})
// var _ = fs.HandleReadAller(&FileHandle{})
var _ = fs.HandleReader(&FileHandle{})
var _ = fs.HandleFlusher(&FileHandle{})

View file

@ -3,9 +3,9 @@ package filesys
import (
"bazil.org/fuse/fs"
"fmt"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/karlseguin/ccache"
"google.golang.org/grpc"
)
type WFS struct {
@ -15,7 +15,7 @@ type WFS struct {
func NewSeaweedFileSystem(filer string) *WFS {
return &WFS{
filer: filer,
filer: filer,
listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)),
}
}

View file

@ -181,6 +181,7 @@ type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"favicon": &bintree{nil, map[string]*bintree{
"favicon.ico": &bintree{favicon, map[string]*bintree{}},
@ -233,4 +234,3 @@ func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

View file

@ -1,16 +1,16 @@
package weed_server
import (
"fmt"
"context"
"time"
"fmt"
"os"
"path/filepath"
"time"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {

View file

@ -7,10 +7,6 @@ import (
"sync"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filer2"
_ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra"
_ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
@ -18,6 +14,10 @@ import (
_ "github.com/chrislusf/seaweedfs/weed/filer2/mysql"
_ "github.com/chrislusf/seaweedfs/weed/filer2/postgres"
_ "github.com/chrislusf/seaweedfs/weed/filer2/redis"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
)
type FilerServer struct {

View file

@ -3,10 +3,10 @@ package weed_server
import (
"net/http"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/filer2"
"strconv"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"strconv"
"time"
)

View file

@ -7,11 +7,11 @@ import (
"strconv"
"strings"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filer2"
)
// listDirectoryHandler lists directories and folers under a directory

View file

@ -17,13 +17,12 @@ import (
"strconv"
"strings"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"time"
)

View file

@ -1,8 +1,8 @@
package master_ui
import (
"strings"
"path/filepath"
"strings"
)
type Breadcrumb struct {

View file

@ -198,7 +198,7 @@ func postFollowingOneRedirect(target string, contentType string, b bytes.Buffer)
reply := string(data)
if strings.HasPrefix(reply, "\"http") {
urlStr := reply[1: len(reply)-1]
urlStr := reply[1 : len(reply)-1]
glog.V(0).Infoln("Post redirected to ", urlStr)
resp2, err2 := http.Post(urlStr, contentType, backupReader)

View file

@ -8,8 +8,8 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
)
func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -5,9 +5,9 @@ import (
"path/filepath"
"time"
ui "github.com/chrislusf/seaweedfs/weed/server/volume_server_ui"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
ui "github.com/chrislusf/seaweedfs/weed/server/volume_server_ui"
)
func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) {

View file

@ -86,7 +86,7 @@ func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) e
for count > 0 && e == nil || e == io.EOF {
for i = 0; i+16 <= count; i += 16 {
key, offset, size = idxFileEntry(bytes[i: i+16])
key, offset, size = idxFileEntry(bytes[i : i+16])
if e = fn(key, offset, size); e != nil {
return e
}

View file

@ -12,7 +12,7 @@ func getActualSize(size uint32) int64 {
return NeedleHeaderSize + int64(size) + NeedleChecksumSize + int64(padding)
}
func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (error) {
func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error {
var indexSize int64
var e error
if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil {