mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
load filer conf and match by prefix
This commit is contained in:
Chris Lu
parent
68043cfcac
commit
0a406f652e
5
go.mod
5
go.mod
|
|
@ -24,6 +24,7 @@ require (
|
|||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.8.0
|
||||
github.com/frankban/quicktest v1.7.2 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis v6.15.7+incompatible
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
|
||||
|
|
@ -45,6 +46,7 @@ require (
|
|||
github.com/klauspost/reedsolomon v1.9.2
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.2.0
|
||||
github.com/lunixbochs/vtclean v1.0.0 // indirect
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
|
||||
github.com/mattn/go-runewidth v0.0.4 // indirect
|
||||
|
|
@ -67,6 +69,9 @@ require (
|
|||
github.com/tidwall/gjson v1.3.2
|
||||
github.com/tidwall/match v1.0.1
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/viant/assertly v0.5.4 // indirect
|
||||
github.com/viant/ptrie v0.3.0
|
||||
github.com/viant/toolbox v0.33.2 // indirect
|
||||
github.com/willf/bitset v1.1.10 // indirect
|
||||
github.com/willf/bloom v2.0.3+incompatible
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -84,10 +84,6 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
|||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011 h1:vN1GvfLgDg8kIPCdhuVKAjlYpxG1B86jiKejB6MC/Q0=
|
||||
github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
|
||||
github.com/chrislusf/raft v1.0.2 h1:3QOYYoZ8HFuJgwmhaIuPIJ1aFep4HoC3fMQYgnf2vms=
|
||||
github.com/chrislusf/raft v1.0.2/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
|
||||
github.com/chrislusf/raft v1.0.3 h1:11YrnzJtVa5z7m9lhY2p8VcPHoUlC1UswyoAo+U1m1k=
|
||||
github.com/chrislusf/raft v1.0.3/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
|
||||
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
|
||||
|
|
@ -163,6 +159,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
|
|||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
|
||||
github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
|
||||
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
|
|
@ -390,6 +388,8 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
|
|||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
|
||||
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
|
||||
github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
|
|
@ -625,6 +625,12 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
|
|||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/viant/assertly v0.5.4 h1:5Hh4U3pLZa6uhCFAGpYOxck/8l9TZczEzoHNfJAhHEQ=
|
||||
github.com/viant/assertly v0.5.4/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/ptrie v0.3.0 h1:SDaRd7Gqr1+ItCNz0GpTxRdK21nOfqjV6YtBm9jGlMY=
|
||||
github.com/viant/ptrie v0.3.0/go.mod h1:VguMnbGfz95Zw+V5VarYSqtqslDxJbOv++xLzxkMhec=
|
||||
github.com/viant/toolbox v0.33.2 h1:Av844IIeGz81gT672qZemyptGfbrcxqGymA5RFnIPjE=
|
||||
github.com/viant/toolbox v0.33.2/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
|
||||
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
|
||||
|
|
|
|||
48
weed/filer/filer_conf.go
Normal file
48
weed/filer/filer_conf.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/viant/ptrie"
|
||||
)
|
||||
|
||||
type FilerConf struct {
|
||||
rules ptrie.Trie
|
||||
}
|
||||
|
||||
func NewFilerConf(data []byte) (fc *FilerConf) {
|
||||
fc = &FilerConf{
|
||||
rules: ptrie.New(),
|
||||
}
|
||||
|
||||
conf := &filer_pb.FilerConf{}
|
||||
err := proto.UnmarshalText(string(data), conf)
|
||||
if err != nil {
|
||||
glog.Errorf("unable to parse filer conf: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
fc.doLoadConf(conf)
|
||||
return fc
|
||||
}
|
||||
|
||||
func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) {
|
||||
for _, location := range conf.Locations {
|
||||
err := fc.rules.Put([]byte(location.LocationPrefix), location)
|
||||
if err != nil {
|
||||
glog.Errorf("put location prefix: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf){
|
||||
fc.rules.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool {
|
||||
pathConf = value.(*filer_pb.FilerConf_PathConf)
|
||||
return true
|
||||
})
|
||||
if pathConf == nil {
|
||||
return &filer_pb.FilerConf_PathConf{}
|
||||
}
|
||||
return pathConf
|
||||
}
|
||||
31
weed/filer/filer_conf_test.go
Normal file
31
weed/filer/filer_conf_test.go
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/viant/ptrie"
|
||||
)
|
||||
|
||||
func TestFilerConf(t *testing.T) {
|
||||
|
||||
fc := &FilerConf{
|
||||
rules: ptrie.New(),
|
||||
}
|
||||
conf := &filer_pb.FilerConf{Locations: []*filer_pb.FilerConf_PathConf{
|
||||
{
|
||||
LocationPrefix: "/buckets/abc",
|
||||
Collection: "abc",
|
||||
},
|
||||
{
|
||||
LocationPrefix: "/buckets/abcd",
|
||||
Collection: "abcd",
|
||||
},
|
||||
}}
|
||||
fc.doLoadConf(conf)
|
||||
|
||||
assert.Equal(t, "abc", fc.MatchStorageRule("/buckets/abc/jasdf").Collection)
|
||||
assert.Equal(t, "abcd", fc.MatchStorageRule("/buckets/abcd/jasdf").Collection)
|
||||
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ import (
|
|||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/viant/ptrie"
|
||||
)
|
||||
|
||||
func ToFileIdObject(fileIdStr string) (*FileId, error) {
|
||||
|
|
@ -138,3 +139,8 @@ func IsRename(event *SubscribeMetadataResponse) bool {
|
|||
event.EventNotification.OldEntry != nil &&
|
||||
event.Directory != event.EventNotification.NewParentPath
|
||||
}
|
||||
|
||||
var _ = ptrie.KeyProvider(&FilerConf_PathConf{})
|
||||
func (fp *FilerConf_PathConf) Key() interface{} {
|
||||
return fp.LocationPrefix
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue