mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge branch 'add_rocksdb'
This commit is contained in:
commit
5f538c5f9d
1
go.mod
1
go.mod
|
@ -66,6 +66,7 @@ require (
|
|||
github.com/spf13/viper v1.4.0
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/syndtr/goleveldb v1.0.0
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
|
||||
github.com/tidwall/gjson v1.3.2
|
||||
github.com/tidwall/match v1.0.1
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
|
|
8
go.sum
8
go.sum
|
@ -225,6 +225,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er
|
|||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
|
@ -542,6 +543,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhD
|
|||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
|
@ -551,8 +553,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
|
|||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/seaweedfs/fuse v1.0.7 h1:tESMXhI3gXzN+dlWsCUrkIZDiWA4dZX18rQMoqmvazw=
|
||||
github.com/seaweedfs/fuse v1.0.7/go.mod h1:W7ubwr1l7KQsMeUpxFFOFOSxUL/ucTRMAlVYs4xdfQ8=
|
||||
github.com/seaweedfs/fuse v1.0.8 h1:HBPJTC77OlxwSd2JiTwvLPn8bWTElqQp3xs9vf3C15s=
|
||||
github.com/seaweedfs/fuse v1.0.8/go.mod h1:W7ubwr1l7KQsMeUpxFFOFOSxUL/ucTRMAlVYs4xdfQ8=
|
||||
github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
|
||||
|
@ -618,6 +618,8 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
|
|||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
|
||||
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
|
||||
github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI=
|
||||
github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
|
||||
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
|
||||
|
@ -941,7 +943,9 @@ honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXe
|
|||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
modernc.org/b v1.0.0 h1:vpvqeyp17ddcQWF29Czawql4lDdABCDRbXRAS4+aF2o=
|
||||
modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
|
||||
modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE=
|
||||
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
|
|
|
@ -87,7 +87,13 @@ buckets_folder = "/buckets"
|
|||
# local on disk, mostly for simple single-machine setup, fairly scalable
|
||||
# faster than previous leveldb, recommended.
|
||||
enabled = true
|
||||
dir = "." # directory to store level db files
|
||||
dir = "./filerldb2" # directory to store level db files
|
||||
|
||||
[rocksdb]
|
||||
# local on disk, similar to leveldb
|
||||
# since it is using a C wrapper, you need to install rocksdb and build it by yourself
|
||||
enabled = false
|
||||
dir = "./filerrdb" # directory to store rocksdb files
|
||||
|
||||
[mysql] # or tidb
|
||||
# CREATE TABLE IF NOT EXISTS filemeta (
|
||||
|
|
|
@ -2,9 +2,11 @@ package leveldb
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
|
@ -86,3 +88,28 @@ func TestEmptyRoot(t *testing.T) {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
func BenchmarkInsertEntry(b *testing.B) {
|
||||
testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
|
||||
dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench")
|
||||
defer os.RemoveAll(dir)
|
||||
store := &LevelDBStore{}
|
||||
store.initialize(dir)
|
||||
testFiler.SetStore(store)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
b.ReportAllocs()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
entry := &filer.Entry{
|
||||
FullPath: util.FullPath(fmt.Sprintf("/file%d.txt", i)),
|
||||
Attr: filer.Attr{
|
||||
Crtime: time.Now(),
|
||||
Mtime: time.Now(),
|
||||
Mode: os.FileMode(0644),
|
||||
},
|
||||
}
|
||||
store.InsertEntry(ctx, entry)
|
||||
}
|
||||
}
|
||||
|
|
41
weed/filer/rocksdb/README.md
Normal file
41
weed/filer/rocksdb/README.md
Normal file
|
@ -0,0 +1,41 @@
|
|||
# Prepare the compilation environment on linux
|
||||
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
|
||||
- sudo apt-get update -qq
|
||||
- sudo apt-get install gcc-6 g++-6 libsnappy-dev zlib1g-dev libbz2-dev -qq
|
||||
- export CXX="g++-6" CC="gcc-6"
|
||||
|
||||
- wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags2_2.0-1.1ubuntu1_amd64.deb
|
||||
- sudo dpkg -i libgflags2_2.0-1.1ubuntu1_amd64.deb
|
||||
- wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags-dev_2.0-1.1ubuntu1_amd64.deb
|
||||
- sudo dpkg -i libgflags-dev_2.0-1.1ubuntu1_amd64.deb
|
||||
|
||||
# Prepare the compilation environment on mac os
|
||||
```
|
||||
brew install snappy
|
||||
```
|
||||
|
||||
# install rocksdb:
|
||||
```
|
||||
export ROCKSDB_HOME=/Users/chris/dev/rocksdb
|
||||
|
||||
git clone https://github.com/facebook/rocksdb.git $ROCKSDB_HOME
|
||||
pushd $ROCKSDB_HOME
|
||||
make clean
|
||||
make install-static
|
||||
popd
|
||||
```
|
||||
|
||||
# install gorocksdb
|
||||
|
||||
```
|
||||
export CGO_CFLAGS="-I$ROCKSDB_HOME/include"
|
||||
export CGO_LDFLAGS="-L$ROCKSDB_HOME -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
|
||||
|
||||
go get github.com/tecbot/gorocksdb
|
||||
```
|
||||
# compile with rocksdb
|
||||
|
||||
```
|
||||
cd ~/go/src/github.com/chrislusf/seaweedfs/weed
|
||||
go install -tags rocksdb
|
||||
```
|
293
weed/filer/rocksdb/rocksdb_store.go
Normal file
293
weed/filer/rocksdb/rocksdb_store.go
Normal file
|
@ -0,0 +1,293 @@
|
|||
// +build rocksdb
|
||||
|
||||
package rocksdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
weed_util "github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/tecbot/gorocksdb"
|
||||
"io"
|
||||
)
|
||||
|
||||
func init() {
|
||||
filer.Stores = append(filer.Stores, &RocksDBStore{})
|
||||
}
|
||||
|
||||
type RocksDBStore struct {
|
||||
path string
|
||||
db *gorocksdb.DB
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) GetName() string {
|
||||
return "rocksdb"
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
|
||||
dir := configuration.GetString(prefix + "dir")
|
||||
return store.initialize(dir)
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) initialize(dir string) (err error) {
|
||||
glog.Infof("filer store rocksdb dir: %s", dir)
|
||||
if err := weed_util.TestFolderWritable(dir); err != nil {
|
||||
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
|
||||
}
|
||||
|
||||
options := gorocksdb.NewDefaultOptions()
|
||||
options.SetCreateIfMissing(true)
|
||||
store.db, err = gorocksdb.OpenDb(options, dir)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
|
||||
return ctx, nil
|
||||
}
|
||||
func (store *RocksDBStore) CommitTransaction(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
func (store *RocksDBStore) RollbackTransaction(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
dir, name := entry.DirAndName()
|
||||
key := genKey(dir, name)
|
||||
|
||||
value, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
|
||||
}
|
||||
|
||||
wo := gorocksdb.NewDefaultWriteOptions()
|
||||
err = store.db.Put(wo, key, value)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
// println("saved", entry.FullPath, "chunks", len(entry.Chunks))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
return store.InsertEntry(ctx, entry)
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
|
||||
dir, name := fullpath.DirAndName()
|
||||
key := genKey(dir, name)
|
||||
|
||||
ro := gorocksdb.NewDefaultReadOptions()
|
||||
data, err := store.db.GetBytes(ro, key)
|
||||
|
||||
if data == nil {
|
||||
return nil, filer_pb.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
entry = &filer.Entry{
|
||||
FullPath: fullpath,
|
||||
}
|
||||
err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(data))
|
||||
if err != nil {
|
||||
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
|
||||
dir, name := fullpath.DirAndName()
|
||||
key := genKey(dir, name)
|
||||
|
||||
wo := gorocksdb.NewDefaultWriteOptions()
|
||||
err = store.db.Delete(wo, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
|
||||
directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
|
||||
|
||||
batch := new(gorocksdb.WriteBatch)
|
||||
|
||||
ro := gorocksdb.NewDefaultReadOptions()
|
||||
ro.SetFillCache(false)
|
||||
iter := store.db.NewIterator(ro)
|
||||
defer iter.Close()
|
||||
err = enumerate(iter, directoryPrefix, nil, false, -1, func(key, value []byte) bool {
|
||||
batch.Delete(key)
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete list %s : %v", fullpath, err)
|
||||
}
|
||||
|
||||
wo := gorocksdb.NewDefaultWriteOptions()
|
||||
err = store.db.Write(wo, batch)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func enumerate(iter *gorocksdb.Iterator, prefix, lastKey []byte, includeLastKey bool, limit int, fn func(key, value []byte) bool) error {
|
||||
|
||||
if len(lastKey) == 0 {
|
||||
iter.Seek(prefix)
|
||||
} else {
|
||||
iter.Seek(lastKey)
|
||||
|
||||
if !includeLastKey {
|
||||
k := iter.Key()
|
||||
v := iter.Value()
|
||||
key := k.Data()
|
||||
defer k.Free()
|
||||
defer v.Free()
|
||||
|
||||
if !bytes.HasPrefix(key, prefix) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if bytes.Equal(key, lastKey) {
|
||||
iter.Next()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
i := 0
|
||||
for ; iter.Valid(); iter.Next() {
|
||||
|
||||
if limit > 0 {
|
||||
i++
|
||||
if i > limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
k := iter.Key()
|
||||
v := iter.Value()
|
||||
key := k.Data()
|
||||
value := v.Data()
|
||||
|
||||
if !bytes.HasPrefix(key, prefix) {
|
||||
k.Free()
|
||||
v.Free()
|
||||
break
|
||||
}
|
||||
|
||||
ret := fn(key, value)
|
||||
|
||||
k.Free()
|
||||
v.Free()
|
||||
|
||||
if !ret {
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if err := iter.Err(); err != nil {
|
||||
return fmt.Errorf("prefix scan iterator: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) ListDirectoryEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool,
|
||||
limit int) (entries []*filer.Entry, err error) {
|
||||
return store.ListDirectoryPrefixedEntries(ctx, fullpath, startFileName, inclusive, limit, "")
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int, prefix string) (entries []*filer.Entry, err error) {
|
||||
|
||||
directoryPrefix := genDirectoryKeyPrefix(fullpath, prefix)
|
||||
lastFileStart := directoryPrefix
|
||||
if startFileName != "" {
|
||||
lastFileStart = genDirectoryKeyPrefix(fullpath, startFileName)
|
||||
}
|
||||
|
||||
ro := gorocksdb.NewDefaultReadOptions()
|
||||
ro.SetFillCache(false)
|
||||
iter := store.db.NewIterator(ro)
|
||||
defer iter.Close()
|
||||
err = enumerate(iter, directoryPrefix, lastFileStart, inclusive, limit, func(key, value []byte) bool {
|
||||
fileName := getNameFromKey(key)
|
||||
if fileName == "" {
|
||||
return true
|
||||
}
|
||||
limit--
|
||||
if limit < 0 {
|
||||
return false
|
||||
}
|
||||
entry := &filer.Entry{
|
||||
FullPath: weed_util.NewFullPath(string(fullpath), fileName),
|
||||
}
|
||||
|
||||
// println("list", entry.FullPath, "chunks", len(entry.Chunks))
|
||||
if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(value)); decodeErr != nil {
|
||||
err = decodeErr
|
||||
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
|
||||
return false
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return entries, fmt.Errorf("prefix list %s : %v", fullpath, err)
|
||||
}
|
||||
|
||||
return entries, err
|
||||
}
|
||||
|
||||
func genKey(dirPath, fileName string) (key []byte) {
|
||||
key = hashToBytes(dirPath)
|
||||
key = append(key, []byte(fileName)...)
|
||||
return key
|
||||
}
|
||||
|
||||
func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) {
|
||||
keyPrefix = hashToBytes(string(fullpath))
|
||||
if len(startFileName) > 0 {
|
||||
keyPrefix = append(keyPrefix, []byte(startFileName)...)
|
||||
}
|
||||
return keyPrefix
|
||||
}
|
||||
|
||||
func getNameFromKey(key []byte) string {
|
||||
|
||||
return string(key[md5.Size:])
|
||||
|
||||
}
|
||||
|
||||
// hash directory, and use last byte for partitioning
|
||||
func hashToBytes(dir string) []byte {
|
||||
h := md5.New()
|
||||
io.WriteString(h, dir)
|
||||
|
||||
b := h.Sum(nil)
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) Shutdown() {
|
||||
store.db.Close()
|
||||
}
|
51
weed/filer/rocksdb/rocksdb_store_kv.go
Normal file
51
weed/filer/rocksdb/rocksdb_store_kv.go
Normal file
|
@ -0,0 +1,51 @@
|
|||
// +build rocksdb
|
||||
|
||||
package rocksdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/tecbot/gorocksdb"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
)
|
||||
|
||||
func (store *RocksDBStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
|
||||
|
||||
wo := gorocksdb.NewDefaultWriteOptions()
|
||||
err = store.db.Put(wo, key, value)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("kv put: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
|
||||
|
||||
ro := gorocksdb.NewDefaultReadOptions()
|
||||
value, err = store.db.GetBytes(ro, key)
|
||||
|
||||
if value == nil {
|
||||
return nil, filer.ErrKvNotFound
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kv get: %v", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (store *RocksDBStore) KvDelete(ctx context.Context, key []byte) (err error) {
|
||||
|
||||
wo := gorocksdb.NewDefaultWriteOptions()
|
||||
err = store.db.Delete(wo, key)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("kv delete: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
117
weed/filer/rocksdb/rocksdb_store_test.go
Normal file
117
weed/filer/rocksdb/rocksdb_store_test.go
Normal file
|
@ -0,0 +1,117 @@
|
|||
// +build rocksdb
|
||||
|
||||
package rocksdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func TestCreateAndFind(t *testing.T) {
|
||||
testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
|
||||
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
|
||||
defer os.RemoveAll(dir)
|
||||
store := &RocksDBStore{}
|
||||
store.initialize(dir)
|
||||
testFiler.SetStore(store)
|
||||
|
||||
fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
entry1 := &filer.Entry{
|
||||
FullPath: fullpath,
|
||||
Attr: filer.Attr{
|
||||
Mode: 0440,
|
||||
Uid: 1234,
|
||||
Gid: 5678,
|
||||
},
|
||||
}
|
||||
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
|
||||
t.Errorf("create entry %v: %v", entry1.FullPath, err)
|
||||
return
|
||||
}
|
||||
|
||||
entry, err := testFiler.FindEntry(ctx, fullpath)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("find entry: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if entry.FullPath != entry1.FullPath {
|
||||
t.Errorf("find wrong entry: %v", entry.FullPath)
|
||||
return
|
||||
}
|
||||
|
||||
// checking one upper directory
|
||||
entries, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "")
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("list entries count: %v", len(entries))
|
||||
return
|
||||
}
|
||||
|
||||
// checking one upper directory
|
||||
entries, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("list entries count: %v", len(entries))
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
|
||||
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
|
||||
defer os.RemoveAll(dir)
|
||||
store := &RocksDBStore{}
|
||||
store.initialize(dir)
|
||||
testFiler.SetStore(store)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// checking one upper directory
|
||||
entries, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "")
|
||||
if err != nil {
|
||||
t.Errorf("list entries: %v", err)
|
||||
return
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("list entries count: %v", len(entries))
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func BenchmarkInsertEntry(b *testing.B) {
|
||||
testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
|
||||
dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench")
|
||||
defer os.RemoveAll(dir)
|
||||
store := &RocksDBStore{}
|
||||
store.initialize(dir)
|
||||
testFiler.SetStore(store)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
b.ReportAllocs()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
entry := &filer.Entry{
|
||||
FullPath: util.FullPath(fmt.Sprintf("/file%d.txt", i)),
|
||||
Attr: filer.Attr{
|
||||
Crtime: time.Now(),
|
||||
Mtime: time.Now(),
|
||||
Mode: os.FileMode(0644),
|
||||
},
|
||||
}
|
||||
store.InsertEntry(ctx, entry)
|
||||
}
|
||||
}
|
7
weed/server/filer_server_rocksdb.go
Normal file
7
weed/server/filer_server_rocksdb.go
Normal file
|
@ -0,0 +1,7 @@
|
|||
// +build rocksdb
|
||||
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/rocksdb"
|
||||
)
|
Loading…
Reference in a new issue