seaweedfs/weed/server/filer_server.go

214 lines
7.2 KiB
Go
Raw Normal View History

2014-03-30 18:28:04 +00:00
package weed_server
import (
2019-06-23 22:29:49 +00:00
"context"
"fmt"
2018-08-13 08:22:32 +00:00
"net/http"
"os"
2020-03-30 08:19:33 +00:00
"sync"
2019-06-23 22:29:49 +00:00
"time"
2018-08-13 08:22:32 +00:00
"github.com/chrislusf/seaweedfs/weed/stats"
"google.golang.org/grpc"
2020-05-05 09:05:28 +00:00
"github.com/chrislusf/seaweedfs/weed/util/grace"
2019-06-23 22:29:49 +00:00
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
2019-06-23 22:29:49 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/util"
2020-09-01 07:21:19 +00:00
"github.com/chrislusf/seaweedfs/weed/filer"
2022-03-17 09:49:26 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/arangodb"
2020-09-01 07:21:19 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
2020-09-03 09:05:26 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
2020-09-01 07:21:19 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
2020-12-24 05:49:01 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/hbase"
2020-09-01 07:21:19 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3"
2020-09-01 07:21:19 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/mongodb"
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql"
2021-01-20 01:21:50 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql2"
2020-09-01 07:21:19 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres"
2021-01-20 02:07:29 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres2"
2020-09-01 07:21:19 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
2021-10-04 08:01:31 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/redis3"
2021-07-01 08:21:14 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
2022-05-01 21:07:47 +00:00
_ "github.com/chrislusf/seaweedfs/weed/filer/ydb"
2018-08-13 08:22:32 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2018-09-16 08:18:30 +00:00
"github.com/chrislusf/seaweedfs/weed/notification"
_ "github.com/chrislusf/seaweedfs/weed/notification/aws_sqs"
2019-07-17 08:24:20 +00:00
_ "github.com/chrislusf/seaweedfs/weed/notification/gocdk_pub_sub"
_ "github.com/chrislusf/seaweedfs/weed/notification/google_pub_sub"
2018-11-01 08:12:21 +00:00
_ "github.com/chrislusf/seaweedfs/weed/notification/kafka"
2018-09-16 08:18:30 +00:00
_ "github.com/chrislusf/seaweedfs/weed/notification/log"
2018-07-22 00:39:10 +00:00
"github.com/chrislusf/seaweedfs/weed/security"
2014-03-30 18:28:04 +00:00
)
2018-07-07 09:18:47 +00:00
type FilerOption struct {
Masters map[string]pb.ServerAddress
2022-05-02 04:59:16 +00:00
FilerGroup string
Collection string
DefaultReplication string
DisableDirListing bool
MaxMB int
DirListingLimit int
DataCenter string
Rack string
DataNode string
DefaultLevelDbDir string
DisableHttp bool
Host pb.ServerAddress
recursiveDelete bool
Cipher bool
SaveToFilerLimit int64
ConcurrentUploadLimit int64
ShowUIDirectoryDelete bool
2018-07-07 09:18:47 +00:00
}
2014-03-30 18:28:04 +00:00
type FilerServer struct {
filer_pb.UnimplementedSeaweedFilerServer
2019-02-18 20:11:52 +00:00
option *FilerOption
secret security.SigningKey
2020-09-01 07:21:19 +00:00
filer *filer.Filer
FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.*` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does **not work** with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. **This now works with the [S3 Gateway](Amazon S3 API).** ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158
2021-12-29 18:47:53 +00:00
filerGuard *security.Guard
2019-02-18 20:11:52 +00:00
grpcDialOption grpc.DialOption
2020-03-30 08:19:33 +00:00
2020-09-17 13:43:54 +00:00
// metrics read from the master
metricsAddress string
metricsIntervalSec int
2020-03-30 08:19:33 +00:00
// notifying clients
2020-04-05 07:51:16 +00:00
listenersLock sync.Mutex
listenersCond *sync.Cond
2020-05-05 09:05:28 +00:00
// track known metadata listeners
knownListenersLock sync.Mutex
knownListeners map[int32]struct{}
2020-05-08 09:47:22 +00:00
brokers map[string]map[string]bool
2020-05-05 09:05:28 +00:00
brokersLock sync.Mutex
inFlightDataSize int64
inFlightDataLimitCond *sync.Cond
2014-03-30 18:28:04 +00:00
}
2018-07-07 09:18:47 +00:00
func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) {
2018-10-07 17:54:05 +00:00
FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.*` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does **not work** with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. **This now works with the [S3 Gateway](Amazon S3 API).** ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158
2021-12-29 18:47:53 +00:00
v := util.GetViper()
signingKey := v.GetString("jwt.filer_signing.key")
v.SetDefault("jwt.filer_signing.expires_after_seconds", 10)
expiresAfterSec := v.GetInt("jwt.filer_signing.expires_after_seconds")
readSigningKey := v.GetString("jwt.filer_signing.read.key")
v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60)
readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds")
2014-03-30 18:28:04 +00:00
fs = &FilerServer{
option: option,
grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
knownListeners: make(map[int32]struct{}),
brokers: make(map[string]map[string]bool),
inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
2014-03-30 18:28:04 +00:00
}
2020-04-05 07:51:16 +00:00
fs.listenersCond = sync.NewCond(&fs.listenersLock)
2018-07-07 09:18:47 +00:00
if len(option.Masters) == 0 {
glog.Fatal("master list is required!")
}
2022-05-02 04:59:16 +00:00
fs.filer = filer.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.FilerGroup, option.Collection, option.DefaultReplication, option.DataCenter, func() {
fs.listenersCond.Broadcast()
})
fs.filer.Cipher = option.Cipher
FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.*` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does **not work** with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. **This now works with the [S3 Gateway](Amazon S3 API).** ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158
2021-12-29 18:47:53 +00:00
// we do not support IP whitelist right now
fs.filerGuard = security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
fs.checkWithMaster()
2020-04-03 07:40:54 +00:00
go stats.LoopPushingMetric("filer", string(fs.option.Host), fs.metricsAddress, fs.metricsIntervalSec)
2021-11-06 21:23:35 +00:00
go fs.filer.KeepMasterClientConnected()
if !util.LoadConfiguration("filer", false) {
v.SetDefault("leveldb2.enabled", true)
v.SetDefault("leveldb2.dir", option.DefaultLevelDbDir)
_, err := os.Stat(option.DefaultLevelDbDir)
if os.IsNotExist(err) {
os.MkdirAll(option.DefaultLevelDbDir, 0755)
}
glog.V(0).Infof("default to create filer store dir in %s", option.DefaultLevelDbDir)
} else {
glog.Warningf("skipping default store dir in %s", option.DefaultLevelDbDir)
}
util.LoadConfiguration("notification", false)
2018-05-14 06:56:16 +00:00
fs.option.recursiveDelete = v.GetBool("filer.options.recursive_delete")
2020-04-07 08:58:48 +00:00
v.SetDefault("filer.options.buckets_folder", "/buckets")
2020-04-07 08:30:53 +00:00
fs.filer.DirBucketsPath = v.GetString("filer.options.buckets_folder")
// TODO deprecated, will be be removed after 2020-12-31
2020-11-17 00:57:31 +00:00
// replaced by https://github.com/chrislusf/seaweedfs/wiki/Path-Specific-Configuration
// fs.filer.FsyncBuckets = v.GetStringSlice("filer.options.buckets_fsync")
isFresh := fs.filer.LoadConfiguration(v)
notification.LoadConfiguration(v, "notification.")
2018-08-13 08:20:49 +00:00
2018-10-07 17:54:05 +00:00
handleStaticResources(defaultMux)
if !option.DisableHttp {
defaultMux.HandleFunc("/", fs.filerHandler)
}
2017-05-28 03:14:22 +00:00
if defaultMux != readonlyMux {
handleStaticResources(readonlyMux)
2017-05-28 03:14:22 +00:00
readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
}
2014-03-30 18:28:04 +00:00
2022-05-30 23:28:36 +00:00
existingNodes := fs.filer.ListExistingPeerUpdates()
startFromTime := time.Now().Add(-filer.LogFlushInterval)
if isFresh {
glog.V(0).Infof("%s bootstrap from peers %+v", option.Host, existingNodes)
2022-05-31 18:55:58 +00:00
if err := fs.filer.MaybeBootstrapFromPeers(option.Host, existingNodes, startFromTime); err != nil {
glog.Fatalf("%s bootstrap from %+v", option.Host, existingNodes)
}
}
fs.filer.AggregateFromPeers(option.Host, existingNodes, startFromTime)
2020-11-15 22:06:03 +00:00
fs.filer.LoadFilerConf()
fs.filer.LoadRemoteStorageConfAndMapping()
grace.OnInterrupt(func() {
2020-03-15 03:30:26 +00:00
fs.filer.Shutdown()
})
2019-06-23 22:29:49 +00:00
return fs, nil
}
func (fs *FilerServer) checkWithMaster() {
2019-06-23 22:29:49 +00:00
isConnected := false
for !isConnected {
2020-09-17 13:43:54 +00:00
for _, master := range fs.option.Masters {
readErr := operation.WithMasterServerClient(false, master, fs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
resp, err := masterClient.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return fmt.Errorf("get master %s configuration: %v", master, err)
}
fs.metricsAddress, fs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
if fs.option.DefaultReplication == "" {
fs.option.DefaultReplication = resp.DefaultReplication
}
return nil
})
2020-04-03 07:47:48 +00:00
if readErr == nil {
isConnected = true
} else {
time.Sleep(7 * time.Second)
}
2019-06-23 22:29:49 +00:00
}
}
2014-03-30 18:28:04 +00:00
}