seaweedfs/weed/replication/source/filer_source.go

153 lines
3.8 KiB
Go
Raw Normal View History

2018-09-17 07:27:56 +00:00
package source
import (
"context"
2018-09-21 08:56:43 +00:00
"fmt"
"io"
2018-09-21 08:54:29 +00:00
"net/http"
2018-09-21 08:56:43 +00:00
"strings"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
2018-09-17 07:27:56 +00:00
)
type ReplicationSource interface {
ReadPart(part string) io.ReadCloser
}
type FilerSource struct {
2019-02-18 20:11:52 +00:00
grpcAddress string
grpcDialOption grpc.DialOption
Dir string
address string
proxyByFiler bool
dataCenter string
2018-09-17 07:27:56 +00:00
}
func (fs *FilerSource) Initialize(configuration util.Configuration, prefix string) error {
fs.dataCenter = configuration.GetString(prefix + "dataCenter")
2020-09-09 18:21:23 +00:00
return fs.DoInitialize(
"",
configuration.GetString(prefix+"grpcAddress"),
configuration.GetString(prefix+"directory"),
false,
2018-09-17 07:27:56 +00:00
)
}
func (fs *FilerSource) DoInitialize(address, grpcAddress string, dir string, readChunkFromFiler bool) (err error) {
fs.address = address
if fs.address == "" {
fs.address = pb.GrpcAddressToServerAddress(grpcAddress)
}
2018-09-17 07:27:56 +00:00
fs.grpcAddress = grpcAddress
2018-09-17 08:37:24 +00:00
fs.Dir = dir
fs.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
fs.proxyByFiler = readChunkFromFiler
2018-09-17 07:27:56 +00:00
return nil
}
func (fs *FilerSource) LookupFileId(part string) (fileUrls []string, err error) {
2018-09-17 07:27:56 +00:00
vid2Locations := make(map[string]*filer_pb.Locations)
vid := volumeId(part)
err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
2018-09-17 07:27:56 +00:00
resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
2018-09-17 07:27:56 +00:00
VolumeIds: []string{vid},
})
if err != nil {
return err
}
vid2Locations = resp.LocationsMap
return nil
})
if err != nil {
2018-10-04 06:36:52 +00:00
glog.V(1).Infof("LookupFileId volume id %s: %v", vid, err)
return nil, fmt.Errorf("LookupFileId volume id %s: %v", vid, err)
2018-09-17 07:27:56 +00:00
}
locations := vid2Locations[vid]
if locations == nil || len(locations.Locations) == 0 {
2018-10-04 06:36:52 +00:00
glog.V(1).Infof("LookupFileId locate volume id %s: %v", vid, err)
return nil, fmt.Errorf("LookupFileId locate volume id %s: %v", vid, err)
2018-09-17 07:27:56 +00:00
}
2021-03-01 00:19:47 +00:00
if !fs.proxyByFiler {
for _, loc := range locations.Locations {
fileUrl := fmt.Sprintf("http://%s/%s?readDeleted=true", loc.Url, part)
// Prefer same data center
if fs.dataCenter != "" && fs.dataCenter == loc.DataCenter {
fileUrls = append([]string{fileUrl}, fileUrls...)
} else {
fileUrls = append(fileUrls, fileUrl)
}
2021-03-01 00:19:47 +00:00
}
} else {
fileUrls = append(fileUrls, fmt.Sprintf("http://%s/?proxyChunkId=%s", fs.address, part))
}
2018-10-04 06:36:52 +00:00
return
}
func (fs *FilerSource) ReadPart(fileId string) (filename string, header http.Header, resp *http.Response, err error) {
if fs.proxyByFiler {
2021-08-14 09:55:44 +00:00
return util.DownloadFile("http://"+fs.address+"/?proxyChunkId="+fileId, "")
}
2018-10-04 06:36:52 +00:00
fileUrls, err := fs.LookupFileId(fileId)
2018-10-04 06:36:52 +00:00
if err != nil {
return "", nil, nil, err
}
2018-09-17 07:27:56 +00:00
for _, fileUrl := range fileUrls {
2021-08-13 04:40:33 +00:00
filename, header, resp, err = util.DownloadFile(fileUrl, "")
if err != nil {
glog.V(1).Infof("fail to read from %s: %v", fileUrl, err)
} else {
break
}
}
2018-09-17 07:27:56 +00:00
2020-09-09 10:53:09 +00:00
return filename, header, resp, err
2018-09-17 07:27:56 +00:00
}
2020-04-29 20:26:02 +00:00
var _ = filer_pb.FilerClient(&FilerSource{})
func (fs *FilerSource) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
2018-09-17 07:27:56 +00:00
return pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
2019-04-06 03:31:58 +00:00
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, fs.grpcAddress, false, fs.grpcDialOption)
2018-09-17 07:27:56 +00:00
}
2021-01-28 22:36:29 +00:00
func (fs *FilerSource) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}
func (fs *FilerSource) GetDataCenter() string {
return fs.dataCenter
}
2018-09-17 07:27:56 +00:00
func volumeId(fileId string) string {
lastCommaIndex := strings.LastIndex(fileId, ",")
if lastCommaIndex > 0 {
return fileId[:lastCommaIndex]
}
return fileId
}