seaweedfs/weed/command/s3.go

215 lines
5.9 KiB
Go
Raw Normal View History

2018-07-18 09:37:09 +00:00
package command
import (
"context"
"fmt"
2018-07-18 09:37:09 +00:00
"net/http"
"time"
2020-03-04 08:39:47 +00:00
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/gorilla/mux"
2018-07-18 09:37:09 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2018-07-22 00:39:10 +00:00
"github.com/chrislusf/seaweedfs/weed/s3api"
2020-09-17 13:56:15 +00:00
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
2018-07-18 09:37:09 +00:00
"github.com/chrislusf/seaweedfs/weed/util"
)
var (
s3StandaloneOptions S3Options
2018-07-18 09:37:09 +00:00
)
type S3Options struct {
2020-12-27 18:50:27 +00:00
filer *string
port *int
config *string
domainName *string
tlsPrivateKey *string
tlsCertificate *string
metricsHttpPort *int
allowEmptyFolder *bool
2018-07-18 09:37:09 +00:00
}
func init() {
cmdS3.Run = runS3 // break init cycle
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
2020-10-22 06:23:00 +00:00
s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
2020-02-09 22:30:02 +00:00
s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
2020-09-24 12:45:39 +00:00
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
2020-12-27 18:50:27 +00:00
s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", false, "allow empty folders")
2018-07-18 09:37:09 +00:00
}
var cmdS3 = &Command{
2020-02-09 22:30:02 +00:00
UsageLine: "s3 [-port=8333] [-filer=<ip:port>] [-config=</path/to/config.json>]",
2018-07-18 09:37:09 +00:00
Short: "start a s3 API compatible server that is backed by a filer",
Long: `start a s3 API compatible server that is backed by a filer.
2020-02-09 22:30:02 +00:00
By default, you can use any access key and secret key to access the S3 APIs.
To enable credential based access, create a config.json file similar to this:
{
"identities": [
{
2020-10-08 17:11:59 +00:00
"name": "anonymous",
"actions": [
"Read"
]
},
{
"name": "some_admin_user",
2020-02-09 22:30:02 +00:00
"credentials": [
{
"accessKey": "some_access_key1",
2020-02-10 00:02:05 +00:00
"secretKey": "some_secret_key1"
2020-02-09 22:30:02 +00:00
}
],
"actions": [
"Admin",
"Read",
2020-10-08 17:11:59 +00:00
"List",
"Tagging",
2020-02-09 22:30:02 +00:00
"Write"
]
},
{
"name": "some_read_only_user",
"credentials": [
{
2020-02-10 00:02:05 +00:00
"accessKey": "some_access_key2",
"secretKey": "some_secret_key2"
2020-02-09 22:30:02 +00:00
}
],
"actions": [
"Read"
]
},
{
"name": "some_normal_user",
"credentials": [
{
2020-02-10 00:02:05 +00:00
"accessKey": "some_access_key3",
"secretKey": "some_secret_key3"
2020-02-09 22:30:02 +00:00
}
],
"actions": [
"Read",
2020-10-08 17:11:59 +00:00
"List",
"Tagging",
2020-02-09 22:30:02 +00:00
"Write"
]
2020-02-23 05:34:18 +00:00
},
{
"name": "user_limited_to_bucket1",
"credentials": [
{
"accessKey": "some_access_key4",
"secretKey": "some_secret_key4"
}
],
"actions": [
"Read:bucket1",
2020-10-08 17:11:59 +00:00
"List:bucket1",
"Tagging:bucket1",
2020-02-23 05:34:18 +00:00
"Write:bucket1"
]
2020-02-09 22:30:02 +00:00
}
]
}
2018-07-18 09:37:09 +00:00
`,
}
func runS3(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
2019-02-18 20:11:52 +00:00
2020-09-24 17:21:23 +00:00
go stats_collect.StartMetricsServer(*s3StandaloneOptions.metricsHttpPort)
return s3StandaloneOptions.startS3Server()
}
func (s3opt *S3Options) startS3Server() bool {
2021-03-03 04:59:39 +00:00
filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*s3opt.filer)
2018-07-18 09:37:09 +00:00
if err != nil {
glog.Fatal(err)
return false
}
filerBucketsPath := "/buckets"
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
2020-09-17 13:56:15 +00:00
// metrics read from the filer
var metricsAddress string
var metricsIntervalSec int
2020-02-27 00:49:47 +00:00
for {
2020-03-04 08:39:47 +00:00
err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
2020-02-27 00:49:47 +00:00
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
}
filerBucketsPath = resp.DirBuckets
2020-09-17 13:56:15 +00:00
metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
2020-02-27 00:49:47 +00:00
glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
return nil
})
if err != nil {
2020-02-27 00:49:47 +00:00
glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
time.Sleep(time.Second)
} else {
glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
break
}
}
2020-09-24 17:21:23 +00:00
go stats_collect.LoopPushingMetric("s3", stats_collect.SourceName(uint32(*s3opt.port)), metricsAddress, metricsIntervalSec)
2018-07-18 09:37:09 +00:00
router := mux.NewRouter().SkipClean(true)
_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
Filer: *s3opt.filer,
Port: *s3opt.port,
2018-07-18 09:37:09 +00:00
FilerGrpcAddress: filerGrpcAddress,
2020-02-09 22:30:02 +00:00
Config: *s3opt.config,
DomainName: *s3opt.domainName,
BucketsPath: filerBucketsPath,
GrpcDialOption: grpcDialOption,
2020-12-27 18:50:27 +00:00
AllowEmptyFolder: *s3opt.allowEmptyFolder,
2018-07-18 09:37:09 +00:00
})
if s3ApiServer_err != nil {
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
}
2020-09-24 12:48:39 +00:00
2018-07-23 04:28:54 +00:00
httpS := &http.Server{Handler: router}
listenAddress := fmt.Sprintf(":%d", *s3opt.port)
2018-07-23 04:28:54 +00:00
s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
if err != nil {
glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
2018-07-18 09:37:09 +00:00
}
if *s3opt.tlsPrivateKey != "" {
2020-06-02 07:10:35 +00:00
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
2018-07-23 04:28:54 +00:00
glog.Fatalf("S3 API Server Fail to serve: %v", err)
}
} else {
2020-06-02 07:10:35 +00:00
glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
2018-07-23 04:28:54 +00:00
if err = httpS.Serve(s3ApiListener); err != nil {
glog.Fatalf("S3 API Server Fail to serve: %v", err)
}
2018-07-18 09:37:09 +00:00
}
return true
}