gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

s3: add grpc server to accept configuration changes

Browse source
This commit is contained in:
chrislu 2022-05-15 00:43:37 -07:00
parent 735038b2c1
commit 29198720f2
12 changed files with 386 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
weed/command/filer.go
View file

@ -86,6 +86,7 @@ func init() {
// start s3 on filer
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")

23
weed/command/s3.go
View file

@ -4,11 +4,13 @@ import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"google.golang.org/grpc/reflection"
"net/http"
"time"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/gorilla/mux"
@ -27,6 +29,7 @@ type S3Options struct {
filer *string
bindIp *string
port *int
portGrpc *int
config *string
domainName *string
tlsPrivateKey *string
@ -43,6 +46,7 @@ func init() {
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.")
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port")
s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
@ -179,7 +183,7 @@ func (s3opt *S3Options) startS3Server() bool {
router := mux.NewRouter().SkipClean(true)
_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
s3ApiServer, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
Filer: filerAddress,
Port: *s3opt.port,
Config: *s3opt.config,
@ -196,6 +200,9 @@ func (s3opt *S3Options) startS3Server() bool {
httpS := &http.Server{Handler: router}
if *s3opt.portGrpc == 0 {
*s3opt.portGrpc = 10000 + *s3opt.port
}
if *s3opt.bindIp == "" {
*s3opt.bindIp = "localhost"
}
@ -213,6 +220,20 @@ func (s3opt *S3Options) startS3Server() bool {
}
}
// starting grpc server
grpcPort := *s3opt.portGrpc
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0)
if err != nil {
glog.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err)
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.s3"))
s3_pb.RegisterSeaweedS3Server(grpcS, s3ApiServer)
reflection.Register(grpcS)
if grpcLocalL != nil {
go grpcS.Serve(grpcLocalL)
}
go grpcS.Serve(grpcL)
if *s3opt.tlsPrivateKey != "" {
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
if s3ApiLocalListner != nil {

5
weed/command/scaffold/security.toml
View file

@ -67,6 +67,11 @@ cert = ""
key = ""
allowed_commonNames = "" # comma-separated SSL certificate common names
[grpc.s3]
cert = ""
key = ""
allowed_commonNames = "" # comma-separated SSL certificate common names
[grpc.msg_broker]
cert = ""
key = ""

1
weed/command/server.go
View file

@ -133,6 +133,7 @@ func init() {
serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, "<exprimental> enable tcp port")
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")

1
weed/pb/Makefile
View file

@ -9,6 +9,7 @@ gen:
protoc remote.proto --go_out=./remote_pb --go-grpc_out=./remote_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc iam.proto --go_out=./iam_pb --go-grpc_out=./iam_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc mount.proto --go_out=./mount_pb --go-grpc_out=./mount_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc s3.proto --go_out=./s3_pb --go-grpc_out=./s3_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc messaging.proto --go_out=./messaging_pb --go-grpc_out=./messaging_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
# protoc filer.proto --java_out=../../other/java/client/src/main/java
cp filer.proto ../../other/java/client/src/main/proto

25
weed/pb/s3.proto Normal file
View file

@ -0,0 +1,25 @@
syntax = "proto3";
package messaging_pb;
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/s3_pb";
option java_package = "seaweedfs.client";
option java_outer_classname = "S3Proto";
//////////////////////////////////////////////////
service SeaweedS3 {
rpc Configure (ConfigureRequest) returns (ConfigureResponse) {
}
}
//////////////////////////////////////////////////
message ConfigureRequest {
bytes configuration_file_content = 1;
}
message ConfigureResponse {
}

208
weed/pb/s3_pb/s3.pb.go Normal file
View file

@ -0,0 +1,208 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// source: s3.proto
package s3_pb
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type ConfigureRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
ConfigurationFileContent []byte `protobuf:"bytes,1,opt,name=configuration_file_content,json=configurationFileContent,proto3" json:"configuration_file_content,omitempty"`
}
func (x *ConfigureRequest) Reset() {
*x = ConfigureRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_s3_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ConfigureRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ConfigureRequest) ProtoMessage() {}
func (x *ConfigureRequest) ProtoReflect() protoreflect.Message {
mi := &file_s3_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ConfigureRequest.ProtoReflect.Descriptor instead.
func (*ConfigureRequest) Descriptor() ([]byte, []int) {
return file_s3_proto_rawDescGZIP(), []int{0}
}
func (x *ConfigureRequest) GetConfigurationFileContent() []byte {
if x != nil {
return x.ConfigurationFileContent
}
return nil
}
type ConfigureResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
}
func (x *ConfigureResponse) Reset() {
*x = ConfigureResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_s3_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ConfigureResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ConfigureResponse) ProtoMessage() {}
func (x *ConfigureResponse) ProtoReflect() protoreflect.Message {
mi := &file_s3_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ConfigureResponse.ProtoReflect.Descriptor instead.
func (*ConfigureResponse) Descriptor() ([]byte, []int) {
return file_s3_proto_rawDescGZIP(), []int{1}
}
var File_s3_proto protoreflect.FileDescriptor
var file_s3_proto_rawDesc = []byte{
0x0a, 0x08, 0x73, 0x33, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, 0x65, 0x73, 0x73,
0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x22, 0x50, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x1a,
0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x66, 0x69,
0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
0x52, 0x18, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46,
0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32,
0x5b, 0x0a, 0x09, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x53, 0x33, 0x12, 0x4e, 0x0a, 0x09,
0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x65, 0x73, 0x73,
0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x65, 0x73, 0x73,
0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x49, 0x0a, 0x10,
0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
0x42, 0x07, 0x53, 0x33, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f,
0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70,
0x62, 0x2f, 0x73, 0x33, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_s3_proto_rawDescOnce sync.Once
file_s3_proto_rawDescData = file_s3_proto_rawDesc
)
func file_s3_proto_rawDescGZIP() []byte {
file_s3_proto_rawDescOnce.Do(func() {
file_s3_proto_rawDescData = protoimpl.X.CompressGZIP(file_s3_proto_rawDescData)
})
return file_s3_proto_rawDescData
}
var file_s3_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_s3_proto_goTypes = []interface{}{
(*ConfigureRequest)(nil), // 0: messaging_pb.ConfigureRequest
(*ConfigureResponse)(nil), // 1: messaging_pb.ConfigureResponse
}
var file_s3_proto_depIdxs = []int32{
0, // 0: messaging_pb.SeaweedS3.Configure:input_type -> messaging_pb.ConfigureRequest
1, // 1: messaging_pb.SeaweedS3.Configure:output_type -> messaging_pb.ConfigureResponse
1, // [1:2] is the sub-list for method output_type
0, // [0:1] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_s3_proto_init() }
func file_s3_proto_init() {
if File_s3_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_s3_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ConfigureRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_s3_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ConfigureResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_s3_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_s3_proto_goTypes,
DependencyIndexes: file_s3_proto_depIdxs,
MessageInfos: file_s3_proto_msgTypes,
}.Build()
File_s3_proto = out.File
file_s3_proto_rawDesc = nil
file_s3_proto_goTypes = nil
file_s3_proto_depIdxs = nil
}

101
weed/pb/s3_pb/s3_grpc.pb.go Normal file
View file

@ -0,0 +1,101 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
package s3_pb
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedS3Client is the client API for SeaweedS3 service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type SeaweedS3Client interface {
Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error)
}
type seaweedS3Client struct {
cc grpc.ClientConnInterface
}
func NewSeaweedS3Client(cc grpc.ClientConnInterface) SeaweedS3Client {
return &seaweedS3Client{cc}
}
func (c *seaweedS3Client) Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error) {
out := new(ConfigureResponse)
err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedS3/Configure", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// SeaweedS3Server is the server API for SeaweedS3 service.
// All implementations must embed UnimplementedSeaweedS3Server
// for forward compatibility
type SeaweedS3Server interface {
Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error)
mustEmbedUnimplementedSeaweedS3Server()
}
// UnimplementedSeaweedS3Server must be embedded to have forward compatible implementations.
type UnimplementedSeaweedS3Server struct {
}
func (UnimplementedSeaweedS3Server) Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
}
func (UnimplementedSeaweedS3Server) mustEmbedUnimplementedSeaweedS3Server() {}
// UnsafeSeaweedS3Server may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to SeaweedS3Server will
// result in compilation errors.
type UnsafeSeaweedS3Server interface {
mustEmbedUnimplementedSeaweedS3Server()
}
func RegisterSeaweedS3Server(s grpc.ServiceRegistrar, srv SeaweedS3Server) {
s.RegisterService(&SeaweedS3_ServiceDesc, srv)
}
func _SeaweedS3_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ConfigureRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedS3Server).Configure(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/messaging_pb.SeaweedS3/Configure",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedS3Server).Configure(ctx, req.(*ConfigureRequest))
}
return interceptor(ctx, in, info, handler)
}
// SeaweedS3_ServiceDesc is the grpc.ServiceDesc for SeaweedS3 service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var SeaweedS3_ServiceDesc = grpc.ServiceDesc{
ServiceName: "messaging_pb.SeaweedS3",
HandlerType: (*SeaweedS3Server)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Configure",
Handler: _SeaweedS3_Configure_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "s3.proto",
}

6
weed/s3api/auth_credentials.go
View file

@ -91,7 +91,7 @@ func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3A
if err != nil {
return fmt.Errorf("read S3 config: %v", err)
}
return iam.loadS3ApiConfigurationFromBytes(content)
return iam.LoadS3ApiConfigurationFromBytes(content)
}
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {
@ -100,10 +100,10 @@ func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName str
glog.Warningf("fail to read %s : %v", fileName, readErr)
return fmt.Errorf("fail to read %s : %v", fileName, readErr)
}
return iam.loadS3ApiConfigurationFromBytes(content)
return iam.LoadS3ApiConfigurationFromBytes(content)
}
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromBytes(content []byte) error {
func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromBytes(content []byte) error {
s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
if err := filer.ParseS3ConfigurationFromBytes(content, s3ApiConfiguration); err != nil {
glog.Warningf("unmarshal error: %v", err)

2
weed/s3api/auth_credentials_subscribe.go
View file

@ -23,7 +23,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
dir = message.NewParentPath
}
if dir == filer.IamConfigDirecotry && message.NewEntry.Name == filer.IamIdentityFile {
if err := s3a.iam.loadS3ApiConfigurationFromBytes(message.NewEntry.Content); err != nil {
if err := s3a.iam.LoadS3ApiConfigurationFromBytes(message.NewEntry.Content); err != nil {
return err
}
glog.V(0).Infof("updated %s/%s", filer.IamConfigDirecotry, filer.IamIdentityFile)

2
weed/s3api/s3api_server.go
View file

@ -3,6 +3,7 @@ package s3api
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"net"
"net/http"
"strings"
@ -31,6 +32,7 @@ type S3ApiServerOption struct {
}
type S3ApiServer struct {
s3_pb.UnimplementedSeaweedS3Server
option *S3ApiServerOption
iam *IdentityAccessManagement
randomClientId int32

16
weed/s3api/s3api_server_grpc.go Normal file
View file

@ -0,0 +1,16 @@
package s3api
import (
"context"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
)
func (s3a *S3ApiServer) Configure(ctx context.Context, request *s3_pb.ConfigureRequest) (*s3_pb.ConfigureResponse, error) {
if err := s3a.iam.LoadS3ApiConfigurationFromBytes(request.ConfigurationFileContent); err != nil {
return nil, err
}
return &s3_pb.ConfigureResponse{}, nil
}