seaweedfs/weed/server/volume_grpc_remote.go

84 lines
2.6 KiB
Go
Raw Normal View History

2021-08-07 21:18:53 +00:00
package weed_server
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
2021-09-07 01:30:44 +00:00
"sync"
2021-09-08 22:54:55 +00:00
"time"
2021-08-07 21:18:53 +00:00
)
func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_server_pb.FetchAndWriteNeedleRequest) (resp *volume_server_pb.FetchAndWriteNeedleResponse, err error) {
resp = &volume_server_pb.FetchAndWriteNeedleResponse{}
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
}
2021-08-26 22:18:34 +00:00
remoteConf := req.RemoteConf
2021-08-07 21:18:53 +00:00
client, getClientErr := remote_storage.GetRemoteStorage(remoteConf)
2021-08-09 05:30:36 +00:00
if getClientErr != nil {
2021-08-07 21:18:53 +00:00
return nil, fmt.Errorf("get remote client: %v", getClientErr)
}
2021-08-26 22:18:34 +00:00
remoteStorageLocation := req.RemoteLocation
2021-08-07 21:18:53 +00:00
data, ReadRemoteErr := client.ReadFile(remoteStorageLocation, req.Offset, req.Size)
if ReadRemoteErr != nil {
return nil, fmt.Errorf("read from remote %+v: %v", remoteStorageLocation, ReadRemoteErr)
}
2021-09-07 01:30:44 +00:00
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
n := new(needle.Needle)
n.Id = types.NeedleId(req.NeedleId)
n.Cookie = types.Cookie(req.Cookie)
n.Data, n.DataSize = data, uint32(len(data))
// copied from *Needle.prepareWriteBuffer()
n.Size = 4 + types.Size(n.DataSize) + 1
n.Checksum = needle.NewCRC(n.Data)
2021-09-08 22:54:55 +00:00
n.LastModified = uint64(time.Now().Unix())
n.SetHasLastModifiedDate()
2021-09-07 01:30:44 +00:00
if _, localWriteErr := vs.store.WriteVolumeNeedle(v.Id, n, true, false); localWriteErr != nil {
if err == nil {
err = fmt.Errorf("local write needle %d size %d: %v", req.NeedleId, req.Size, err)
}
}
}()
2021-09-14 17:37:06 +00:00
if len(req.Replicas) > 0 {
2021-09-07 01:30:44 +00:00
fileId := needle.NewFileId(v.Id, req.NeedleId, req.Cookie)
for _, replica := range req.Replicas {
wg.Add(1)
go func(targetVolumeServer string) {
defer wg.Done()
uploadOption := &operation.UploadOption{
UploadUrl: fmt.Sprintf("http://%s/%s?type=replicate", targetVolumeServer, fileId.String()),
Filename: "",
Cipher: false,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: security.EncodedJwt(req.Auth),
}
if _, replicaWriteErr := operation.UploadData(data, uploadOption); replicaWriteErr != nil {
if err == nil {
err = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, err)
}
}
}(replica.Url)
}
2021-08-07 21:18:53 +00:00
}
2021-09-07 01:30:44 +00:00
wg.Wait()
return resp, err
2021-08-07 21:18:53 +00:00
}