mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Chris Lu
GitHub
commit
d2bb351c88
|
|
@ -88,7 +88,32 @@ func runBackup(cmd *Command, args []string) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
if err := v.Synchronize(volumeServer, grpcDialOption); err != nil {
|
||||
if v.SuperBlock.CompactRevision < uint16(stats.CompactRevision) {
|
||||
if err = v.Compact(0); err != nil {
|
||||
fmt.Printf("Compact Volume before synchronizing %v\n", err)
|
||||
return true
|
||||
}
|
||||
if err = v.CommitCompact(); err != nil {
|
||||
fmt.Printf("Commit Compact before synchronizing %v\n", err)
|
||||
return true
|
||||
}
|
||||
v.SuperBlock.CompactRevision = uint16(stats.CompactRevision)
|
||||
v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
|
||||
}
|
||||
|
||||
if uint64(v.Size()) > stats.TailOffset {
|
||||
// remove the old data
|
||||
v.Destroy()
|
||||
// recreate an empty volume
|
||||
v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0)
|
||||
if err != nil {
|
||||
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
|
||||
return true
|
||||
}
|
||||
}
|
||||
defer v.Close()
|
||||
|
||||
if err := v.Follow(volumeServer, grpcDialOption); err != nil {
|
||||
fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset i
|
|||
nv, ok := needleMap.Get(n.Id)
|
||||
glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
|
||||
n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
|
||||
if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
|
||||
if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && int64(nv.Offset)*types.NeedlePaddingSize == offset {
|
||||
if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
|
||||
glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
|
||||
n.LastModified, newerThanUnix)
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
|
|||
|
||||
func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
|
||||
glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
|
||||
if n.Size > 0 {
|
||||
if n.Size > 0 && n.Size != types.TombstoneFileSize {
|
||||
pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
|
||||
glog.V(2).Infof("saved %d with error %v", n.Size, pe)
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -2,12 +2,8 @@ package operation
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
)
|
||||
|
||||
func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
|
||||
|
|
@ -22,40 +18,3 @@ func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint
|
|||
|
||||
return
|
||||
}
|
||||
|
||||
func GetVolumeIdxEntries(server string, grpcDialOption grpc.DialOption, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
|
||||
|
||||
return WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
|
||||
VolumeId: vid,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var indexFileContent []byte
|
||||
|
||||
for {
|
||||
resp, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read index entries: %v", err)
|
||||
}
|
||||
indexFileContent = append(indexFileContent, resp.IndexFileContent...)
|
||||
}
|
||||
|
||||
dataSize := len(indexFileContent)
|
||||
|
||||
for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
|
||||
line := indexFileContent[idx : idx+NeedleEntrySize]
|
||||
key := BytesToNeedleId(line[:NeedleIdSize])
|
||||
offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
|
||||
size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
|
||||
eachEntryFn(key, offset, size)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,9 +24,7 @@ service VolumeServer {
|
|||
|
||||
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
|
||||
}
|
||||
rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) {
|
||||
}
|
||||
rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) {
|
||||
rpc VolumeFollow (VolumeFollowRequest) returns (stream VolumeFollowResponse) {
|
||||
}
|
||||
|
||||
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
|
||||
|
|
@ -119,21 +117,11 @@ message VolumeSyncStatusResponse {
|
|||
uint64 idx_file_size = 8;
|
||||
}
|
||||
|
||||
message VolumeSyncIndexRequest {
|
||||
message VolumeFollowRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 since = 2;
|
||||
}
|
||||
message VolumeSyncIndexResponse {
|
||||
bytes index_file_content = 1;
|
||||
}
|
||||
|
||||
message VolumeSyncDataRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint32 revision = 2;
|
||||
uint32 offset = 3;
|
||||
uint32 size = 4;
|
||||
string needle_id = 5;
|
||||
}
|
||||
message VolumeSyncDataResponse {
|
||||
message VolumeFollowResponse {
|
||||
bytes file_content = 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,10 +27,8 @@ It has these top-level messages:
|
|||
AssignVolumeResponse
|
||||
VolumeSyncStatusRequest
|
||||
VolumeSyncStatusResponse
|
||||
VolumeSyncIndexRequest
|
||||
VolumeSyncIndexResponse
|
||||
VolumeSyncDataRequest
|
||||
VolumeSyncDataResponse
|
||||
VolumeFollowRequest
|
||||
VolumeFollowResponse
|
||||
VolumeMountRequest
|
||||
VolumeMountResponse
|
||||
VolumeUnmountRequest
|
||||
|
|
@ -420,96 +418,40 @@ func (m *VolumeSyncStatusResponse) GetIdxFileSize() uint64 {
|
|||
return 0
|
||||
}
|
||||
|
||||
type VolumeSyncIndexRequest struct {
|
||||
type VolumeFollowRequest struct {
|
||||
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
|
||||
Since uint64 `protobuf:"varint,2,opt,name=since" json:"since,omitempty"`
|
||||
}
|
||||
|
||||
func (m *VolumeSyncIndexRequest) Reset() { *m = VolumeSyncIndexRequest{} }
|
||||
func (m *VolumeSyncIndexRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeSyncIndexRequest) ProtoMessage() {}
|
||||
func (*VolumeSyncIndexRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
|
||||
func (m *VolumeFollowRequest) Reset() { *m = VolumeFollowRequest{} }
|
||||
func (m *VolumeFollowRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeFollowRequest) ProtoMessage() {}
|
||||
func (*VolumeFollowRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
|
||||
|
||||
func (m *VolumeSyncIndexRequest) GetVolumeId() uint32 {
|
||||
func (m *VolumeFollowRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
return m.VolumeId
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type VolumeSyncIndexResponse struct {
|
||||
IndexFileContent []byte `protobuf:"bytes,1,opt,name=index_file_content,json=indexFileContent,proto3" json:"index_file_content,omitempty"`
|
||||
}
|
||||
|
||||
func (m *VolumeSyncIndexResponse) Reset() { *m = VolumeSyncIndexResponse{} }
|
||||
func (m *VolumeSyncIndexResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeSyncIndexResponse) ProtoMessage() {}
|
||||
func (*VolumeSyncIndexResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
|
||||
|
||||
func (m *VolumeSyncIndexResponse) GetIndexFileContent() []byte {
|
||||
func (m *VolumeFollowRequest) GetSince() uint64 {
|
||||
if m != nil {
|
||||
return m.IndexFileContent
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type VolumeSyncDataRequest struct {
|
||||
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
|
||||
Revision uint32 `protobuf:"varint,2,opt,name=revision" json:"revision,omitempty"`
|
||||
Offset uint32 `protobuf:"varint,3,opt,name=offset" json:"offset,omitempty"`
|
||||
Size uint32 `protobuf:"varint,4,opt,name=size" json:"size,omitempty"`
|
||||
NeedleId string `protobuf:"bytes,5,opt,name=needle_id,json=needleId" json:"needle_id,omitempty"`
|
||||
}
|
||||
|
||||
func (m *VolumeSyncDataRequest) Reset() { *m = VolumeSyncDataRequest{} }
|
||||
func (m *VolumeSyncDataRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeSyncDataRequest) ProtoMessage() {}
|
||||
func (*VolumeSyncDataRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
|
||||
|
||||
func (m *VolumeSyncDataRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
return m.VolumeId
|
||||
return m.Since
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *VolumeSyncDataRequest) GetRevision() uint32 {
|
||||
if m != nil {
|
||||
return m.Revision
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *VolumeSyncDataRequest) GetOffset() uint32 {
|
||||
if m != nil {
|
||||
return m.Offset
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *VolumeSyncDataRequest) GetSize() uint32 {
|
||||
if m != nil {
|
||||
return m.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m *VolumeSyncDataRequest) GetNeedleId() string {
|
||||
if m != nil {
|
||||
return m.NeedleId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type VolumeSyncDataResponse struct {
|
||||
type VolumeFollowResponse struct {
|
||||
FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"`
|
||||
}
|
||||
|
||||
func (m *VolumeSyncDataResponse) Reset() { *m = VolumeSyncDataResponse{} }
|
||||
func (m *VolumeSyncDataResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeSyncDataResponse) ProtoMessage() {}
|
||||
func (*VolumeSyncDataResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
|
||||
func (m *VolumeFollowResponse) Reset() { *m = VolumeFollowResponse{} }
|
||||
func (m *VolumeFollowResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeFollowResponse) ProtoMessage() {}
|
||||
func (*VolumeFollowResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
|
||||
|
||||
func (m *VolumeSyncDataResponse) GetFileContent() []byte {
|
||||
func (m *VolumeFollowResponse) GetFileContent() []byte {
|
||||
if m != nil {
|
||||
return m.FileContent
|
||||
}
|
||||
|
|
@ -523,7 +465,7 @@ type VolumeMountRequest struct {
|
|||
func (m *VolumeMountRequest) Reset() { *m = VolumeMountRequest{} }
|
||||
func (m *VolumeMountRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeMountRequest) ProtoMessage() {}
|
||||
func (*VolumeMountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
|
||||
func (*VolumeMountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
|
||||
|
||||
func (m *VolumeMountRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
|
|
@ -538,7 +480,7 @@ type VolumeMountResponse struct {
|
|||
func (m *VolumeMountResponse) Reset() { *m = VolumeMountResponse{} }
|
||||
func (m *VolumeMountResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeMountResponse) ProtoMessage() {}
|
||||
func (*VolumeMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
|
||||
func (*VolumeMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
|
||||
|
||||
type VolumeUnmountRequest struct {
|
||||
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
|
||||
|
|
@ -547,7 +489,7 @@ type VolumeUnmountRequest struct {
|
|||
func (m *VolumeUnmountRequest) Reset() { *m = VolumeUnmountRequest{} }
|
||||
func (m *VolumeUnmountRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeUnmountRequest) ProtoMessage() {}
|
||||
func (*VolumeUnmountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
|
||||
func (*VolumeUnmountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
|
||||
|
||||
func (m *VolumeUnmountRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
|
|
@ -562,7 +504,7 @@ type VolumeUnmountResponse struct {
|
|||
func (m *VolumeUnmountResponse) Reset() { *m = VolumeUnmountResponse{} }
|
||||
func (m *VolumeUnmountResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeUnmountResponse) ProtoMessage() {}
|
||||
func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
|
||||
func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
|
||||
|
||||
type VolumeDeleteRequest struct {
|
||||
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
|
||||
|
|
@ -571,7 +513,7 @@ type VolumeDeleteRequest struct {
|
|||
func (m *VolumeDeleteRequest) Reset() { *m = VolumeDeleteRequest{} }
|
||||
func (m *VolumeDeleteRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeDeleteRequest) ProtoMessage() {}
|
||||
func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
|
||||
func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
|
||||
|
||||
func (m *VolumeDeleteRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
|
|
@ -586,7 +528,7 @@ type VolumeDeleteResponse struct {
|
|||
func (m *VolumeDeleteResponse) Reset() { *m = VolumeDeleteResponse{} }
|
||||
func (m *VolumeDeleteResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*VolumeDeleteResponse) ProtoMessage() {}
|
||||
func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
|
||||
func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
|
||||
|
||||
type ReplicateVolumeRequest struct {
|
||||
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
|
||||
|
|
@ -599,7 +541,7 @@ type ReplicateVolumeRequest struct {
|
|||
func (m *ReplicateVolumeRequest) Reset() { *m = ReplicateVolumeRequest{} }
|
||||
func (m *ReplicateVolumeRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReplicateVolumeRequest) ProtoMessage() {}
|
||||
func (*ReplicateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
|
||||
func (*ReplicateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
|
||||
|
||||
func (m *ReplicateVolumeRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
|
|
@ -642,7 +584,7 @@ type ReplicateVolumeResponse struct {
|
|||
func (m *ReplicateVolumeResponse) Reset() { *m = ReplicateVolumeResponse{} }
|
||||
func (m *ReplicateVolumeResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReplicateVolumeResponse) ProtoMessage() {}
|
||||
func (*ReplicateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
|
||||
func (*ReplicateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
|
||||
|
||||
type CopyFileRequest struct {
|
||||
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
|
||||
|
|
@ -653,7 +595,7 @@ type CopyFileRequest struct {
|
|||
func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} }
|
||||
func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*CopyFileRequest) ProtoMessage() {}
|
||||
func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
|
||||
func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
|
||||
|
||||
func (m *CopyFileRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
|
|
@ -683,7 +625,7 @@ type CopyFileResponse struct {
|
|||
func (m *CopyFileResponse) Reset() { *m = CopyFileResponse{} }
|
||||
func (m *CopyFileResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*CopyFileResponse) ProtoMessage() {}
|
||||
func (*CopyFileResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
|
||||
func (*CopyFileResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
|
||||
|
||||
func (m *CopyFileResponse) GetFileContent() []byte {
|
||||
if m != nil {
|
||||
|
|
@ -699,7 +641,7 @@ type ReadVolumeFileStatusRequest struct {
|
|||
func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} }
|
||||
func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReadVolumeFileStatusRequest) ProtoMessage() {}
|
||||
func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
|
||||
func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
|
||||
|
||||
func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
|
|
@ -719,7 +661,7 @@ type ReadVolumeFileStatusResponse struct {
|
|||
func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} }
|
||||
func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReadVolumeFileStatusResponse) ProtoMessage() {}
|
||||
func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
|
||||
func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
|
||||
|
||||
func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 {
|
||||
if m != nil {
|
||||
|
|
@ -766,7 +708,7 @@ type DiskStatus struct {
|
|||
func (m *DiskStatus) Reset() { *m = DiskStatus{} }
|
||||
func (m *DiskStatus) String() string { return proto.CompactTextString(m) }
|
||||
func (*DiskStatus) ProtoMessage() {}
|
||||
func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
|
||||
func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
|
||||
|
||||
func (m *DiskStatus) GetDir() string {
|
||||
if m != nil {
|
||||
|
|
@ -809,7 +751,7 @@ type MemStatus struct {
|
|||
func (m *MemStatus) Reset() { *m = MemStatus{} }
|
||||
func (m *MemStatus) String() string { return proto.CompactTextString(m) }
|
||||
func (*MemStatus) ProtoMessage() {}
|
||||
func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} }
|
||||
func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
|
||||
|
||||
func (m *MemStatus) GetGoroutines() int32 {
|
||||
if m != nil {
|
||||
|
|
@ -879,10 +821,8 @@ func init() {
|
|||
proto.RegisterType((*AssignVolumeResponse)(nil), "volume_server_pb.AssignVolumeResponse")
|
||||
proto.RegisterType((*VolumeSyncStatusRequest)(nil), "volume_server_pb.VolumeSyncStatusRequest")
|
||||
proto.RegisterType((*VolumeSyncStatusResponse)(nil), "volume_server_pb.VolumeSyncStatusResponse")
|
||||
proto.RegisterType((*VolumeSyncIndexRequest)(nil), "volume_server_pb.VolumeSyncIndexRequest")
|
||||
proto.RegisterType((*VolumeSyncIndexResponse)(nil), "volume_server_pb.VolumeSyncIndexResponse")
|
||||
proto.RegisterType((*VolumeSyncDataRequest)(nil), "volume_server_pb.VolumeSyncDataRequest")
|
||||
proto.RegisterType((*VolumeSyncDataResponse)(nil), "volume_server_pb.VolumeSyncDataResponse")
|
||||
proto.RegisterType((*VolumeFollowRequest)(nil), "volume_server_pb.VolumeFollowRequest")
|
||||
proto.RegisterType((*VolumeFollowResponse)(nil), "volume_server_pb.VolumeFollowResponse")
|
||||
proto.RegisterType((*VolumeMountRequest)(nil), "volume_server_pb.VolumeMountRequest")
|
||||
proto.RegisterType((*VolumeMountResponse)(nil), "volume_server_pb.VolumeMountResponse")
|
||||
proto.RegisterType((*VolumeUnmountRequest)(nil), "volume_server_pb.VolumeUnmountRequest")
|
||||
|
|
@ -919,8 +859,7 @@ type VolumeServerClient interface {
|
|||
DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
|
||||
AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
|
||||
VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error)
|
||||
VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error)
|
||||
VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error)
|
||||
VolumeFollow(ctx context.Context, in *VolumeFollowRequest, opts ...grpc.CallOption) (VolumeServer_VolumeFollowClient, error)
|
||||
VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error)
|
||||
VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error)
|
||||
VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error)
|
||||
|
|
@ -1009,12 +948,12 @@ func (c *volumeServerClient) VolumeSyncStatus(ctx context.Context, in *VolumeSyn
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error) {
|
||||
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncIndex", opts...)
|
||||
func (c *volumeServerClient) VolumeFollow(ctx context.Context, in *VolumeFollowRequest, opts ...grpc.CallOption) (VolumeServer_VolumeFollowClient, error) {
|
||||
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeFollow", opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &volumeServerVolumeSyncIndexClient{stream}
|
||||
x := &volumeServerVolumeFollowClient{stream}
|
||||
if err := x.ClientStream.SendMsg(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -1024,49 +963,17 @@ func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSync
|
|||
return x, nil
|
||||
}
|
||||
|
||||
type VolumeServer_VolumeSyncIndexClient interface {
|
||||
Recv() (*VolumeSyncIndexResponse, error)
|
||||
type VolumeServer_VolumeFollowClient interface {
|
||||
Recv() (*VolumeFollowResponse, error)
|
||||
grpc.ClientStream
|
||||
}
|
||||
|
||||
type volumeServerVolumeSyncIndexClient struct {
|
||||
type volumeServerVolumeFollowClient struct {
|
||||
grpc.ClientStream
|
||||
}
|
||||
|
||||
func (x *volumeServerVolumeSyncIndexClient) Recv() (*VolumeSyncIndexResponse, error) {
|
||||
m := new(VolumeSyncIndexResponse)
|
||||
if err := x.ClientStream.RecvMsg(m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error) {
|
||||
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncData", opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
x := &volumeServerVolumeSyncDataClient{stream}
|
||||
if err := x.ClientStream.SendMsg(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := x.ClientStream.CloseSend(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
type VolumeServer_VolumeSyncDataClient interface {
|
||||
Recv() (*VolumeSyncDataResponse, error)
|
||||
grpc.ClientStream
|
||||
}
|
||||
|
||||
type volumeServerVolumeSyncDataClient struct {
|
||||
grpc.ClientStream
|
||||
}
|
||||
|
||||
func (x *volumeServerVolumeSyncDataClient) Recv() (*VolumeSyncDataResponse, error) {
|
||||
m := new(VolumeSyncDataResponse)
|
||||
func (x *volumeServerVolumeFollowClient) Recv() (*VolumeFollowResponse, error) {
|
||||
m := new(VolumeFollowResponse)
|
||||
if err := x.ClientStream.RecvMsg(m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -1119,7 +1026,7 @@ func (c *volumeServerClient) ReadVolumeFileStatus(ctx context.Context, in *ReadV
|
|||
}
|
||||
|
||||
func (c *volumeServerClient) CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error) {
|
||||
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[2], c.cc, "/volume_server_pb.VolumeServer/CopyFile", opts...)
|
||||
stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/CopyFile", opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -1162,8 +1069,7 @@ type VolumeServerServer interface {
|
|||
DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
|
||||
AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
|
||||
VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error)
|
||||
VolumeSyncIndex(*VolumeSyncIndexRequest, VolumeServer_VolumeSyncIndexServer) error
|
||||
VolumeSyncData(*VolumeSyncDataRequest, VolumeServer_VolumeSyncDataServer) error
|
||||
VolumeFollow(*VolumeFollowRequest, VolumeServer_VolumeFollowServer) error
|
||||
VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error)
|
||||
VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error)
|
||||
VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error)
|
||||
|
|
@ -1320,45 +1226,24 @@ func _VolumeServer_VolumeSyncStatus_Handler(srv interface{}, ctx context.Context
|
|||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_VolumeSyncIndex_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
m := new(VolumeSyncIndexRequest)
|
||||
func _VolumeServer_VolumeFollow_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
m := new(VolumeFollowRequest)
|
||||
if err := stream.RecvMsg(m); err != nil {
|
||||
return err
|
||||
}
|
||||
return srv.(VolumeServerServer).VolumeSyncIndex(m, &volumeServerVolumeSyncIndexServer{stream})
|
||||
return srv.(VolumeServerServer).VolumeFollow(m, &volumeServerVolumeFollowServer{stream})
|
||||
}
|
||||
|
||||
type VolumeServer_VolumeSyncIndexServer interface {
|
||||
Send(*VolumeSyncIndexResponse) error
|
||||
type VolumeServer_VolumeFollowServer interface {
|
||||
Send(*VolumeFollowResponse) error
|
||||
grpc.ServerStream
|
||||
}
|
||||
|
||||
type volumeServerVolumeSyncIndexServer struct {
|
||||
type volumeServerVolumeFollowServer struct {
|
||||
grpc.ServerStream
|
||||
}
|
||||
|
||||
func (x *volumeServerVolumeSyncIndexServer) Send(m *VolumeSyncIndexResponse) error {
|
||||
return x.ServerStream.SendMsg(m)
|
||||
}
|
||||
|
||||
func _VolumeServer_VolumeSyncData_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
m := new(VolumeSyncDataRequest)
|
||||
if err := stream.RecvMsg(m); err != nil {
|
||||
return err
|
||||
}
|
||||
return srv.(VolumeServerServer).VolumeSyncData(m, &volumeServerVolumeSyncDataServer{stream})
|
||||
}
|
||||
|
||||
type VolumeServer_VolumeSyncDataServer interface {
|
||||
Send(*VolumeSyncDataResponse) error
|
||||
grpc.ServerStream
|
||||
}
|
||||
|
||||
type volumeServerVolumeSyncDataServer struct {
|
||||
grpc.ServerStream
|
||||
}
|
||||
|
||||
func (x *volumeServerVolumeSyncDataServer) Send(m *VolumeSyncDataResponse) error {
|
||||
func (x *volumeServerVolumeFollowServer) Send(m *VolumeFollowResponse) error {
|
||||
return x.ServerStream.SendMsg(m)
|
||||
}
|
||||
|
||||
|
|
@ -1532,13 +1417,8 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
|
|||
},
|
||||
Streams: []grpc.StreamDesc{
|
||||
{
|
||||
StreamName: "VolumeSyncIndex",
|
||||
Handler: _VolumeServer_VolumeSyncIndex_Handler,
|
||||
ServerStreams: true,
|
||||
},
|
||||
{
|
||||
StreamName: "VolumeSyncData",
|
||||
Handler: _VolumeServer_VolumeSyncData_Handler,
|
||||
StreamName: "VolumeFollow",
|
||||
Handler: _VolumeServer_VolumeFollow_Handler,
|
||||
ServerStreams: true,
|
||||
},
|
||||
{
|
||||
|
|
@ -1553,83 +1433,79 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
|
|||
func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
|
||||
|
||||
var fileDescriptor0 = []byte{
|
||||
// 1247 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x73, 0xdb, 0xc4,
|
||||
0x17, 0x8f, 0x62, 0x3b, 0x71, 0x8e, 0xed, 0xc6, 0xff, 0x4d, 0x9a, 0x38, 0x4a, 0xff, 0xc1, 0x15,
|
||||
0x90, 0x3a, 0x6d, 0x1a, 0x20, 0x9d, 0x42, 0x81, 0x17, 0x20, 0x01, 0x26, 0x0f, 0xa5, 0x33, 0x0a,
|
||||
0xed, 0x30, 0x43, 0x67, 0x34, 0x1b, 0x69, 0x9d, 0x88, 0xc8, 0x92, 0xaa, 0x5d, 0x85, 0x84, 0x6f,
|
||||
0xc2, 0x33, 0x2f, 0x7d, 0xe7, 0x03, 0xf1, 0x41, 0x78, 0x61, 0xf6, 0x22, 0x59, 0x37, 0xc7, 0xe2,
|
||||
0xf2, 0xb6, 0x3a, 0x7b, 0xce, 0xef, 0x5c, 0xf6, 0xec, 0xd9, 0x9f, 0x0d, 0x6b, 0x57, 0x81, 0x17,
|
||||
0x4f, 0x88, 0x45, 0x49, 0x74, 0x45, 0xa2, 0x83, 0x30, 0x0a, 0x58, 0x80, 0xfa, 0x39, 0xa1, 0x15,
|
||||
0x9e, 0x19, 0x1f, 0x00, 0xfa, 0x0a, 0x33, 0xfb, 0xe2, 0x98, 0x78, 0x84, 0x11, 0x93, 0xbc, 0x89,
|
||||
0x09, 0x65, 0x68, 0x0b, 0xda, 0x63, 0xd7, 0x23, 0x96, 0xeb, 0xd0, 0x81, 0x36, 0x6c, 0x8c, 0x56,
|
||||
0xcc, 0x65, 0xfe, 0x7d, 0xe2, 0x50, 0xe3, 0x05, 0xac, 0xe5, 0x0c, 0x68, 0x18, 0xf8, 0x94, 0xa0,
|
||||
0x67, 0xb0, 0x1c, 0x11, 0x1a, 0x7b, 0x4c, 0x1a, 0x74, 0x0e, 0x77, 0x0e, 0x8a, 0xbe, 0x0e, 0x52,
|
||||
0x93, 0xd8, 0x63, 0x66, 0xa2, 0x6e, 0xb8, 0xd0, 0xcd, 0x6e, 0xa0, 0x4d, 0x58, 0x56, 0xbe, 0x07,
|
||||
0xda, 0x50, 0x1b, 0xad, 0x98, 0x4b, 0xd2, 0x35, 0xda, 0x80, 0x25, 0xca, 0x30, 0x8b, 0xe9, 0x60,
|
||||
0x71, 0xa8, 0x8d, 0x5a, 0xa6, 0xfa, 0x42, 0xeb, 0xd0, 0x22, 0x51, 0x14, 0x44, 0x83, 0x86, 0x50,
|
||||
0x97, 0x1f, 0x08, 0x41, 0x93, 0xba, 0xbf, 0x90, 0x41, 0x73, 0xa8, 0x8d, 0x7a, 0xa6, 0x58, 0x1b,
|
||||
0xcb, 0xd0, 0xfa, 0x7a, 0x12, 0xb2, 0x1b, 0xe3, 0x13, 0x18, 0xbc, 0xc2, 0x76, 0x1c, 0x4f, 0x5e,
|
||||
0x89, 0x18, 0x8f, 0x2e, 0x88, 0x7d, 0x99, 0xe4, 0xbe, 0x0d, 0x2b, 0x2a, 0x72, 0x15, 0x41, 0xcf,
|
||||
0x6c, 0x4b, 0xc1, 0x89, 0x63, 0x7c, 0x01, 0x5b, 0x15, 0x86, 0xaa, 0x06, 0xef, 0x42, 0xef, 0x1c,
|
||||
0x47, 0x67, 0xf8, 0x9c, 0x58, 0x11, 0x66, 0x6e, 0x20, 0xac, 0x35, 0xb3, 0xab, 0x84, 0x26, 0x97,
|
||||
0x19, 0x3f, 0x82, 0x9e, 0x43, 0x08, 0x26, 0x21, 0xb6, 0x59, 0x1d, 0xe7, 0x68, 0x08, 0x9d, 0x30,
|
||||
0x22, 0xd8, 0xf3, 0x02, 0x1b, 0x33, 0x22, 0xaa, 0xd0, 0x30, 0xb3, 0x22, 0xe3, 0xff, 0xb0, 0x5d,
|
||||
0x09, 0x2e, 0x03, 0x34, 0x9e, 0x15, 0xa2, 0x0f, 0x26, 0x13, 0xb7, 0x96, 0x6b, 0xe3, 0x5e, 0x29,
|
||||
0x6a, 0x61, 0xa9, 0x70, 0x3f, 0x2d, 0xec, 0x7a, 0x04, 0xfb, 0x71, 0x58, 0x0b, 0xb8, 0x18, 0x71,
|
||||
0x62, 0x9a, 0x22, 0x6f, 0xca, 0xe6, 0x38, 0x0a, 0x3c, 0x8f, 0xd8, 0xcc, 0x0d, 0xfc, 0x04, 0x76,
|
||||
0x07, 0xc0, 0x4e, 0x85, 0xaa, 0x55, 0x32, 0x12, 0x43, 0x87, 0x41, 0xd9, 0x54, 0xc1, 0xbe, 0xd5,
|
||||
0x60, 0xed, 0x4b, 0x4a, 0xdd, 0x73, 0x5f, 0xba, 0xad, 0x55, 0xfe, 0xbc, 0xc3, 0xc5, 0xa2, 0xc3,
|
||||
0xe2, 0xf1, 0x34, 0x4a, 0xc7, 0xc3, 0x35, 0x22, 0x12, 0x7a, 0xae, 0x8d, 0x05, 0x44, 0x53, 0x40,
|
||||
0x64, 0x45, 0xa8, 0x0f, 0x0d, 0xc6, 0xbc, 0x41, 0x4b, 0xec, 0xf0, 0xa5, 0xb1, 0x01, 0xeb, 0xf9,
|
||||
0x48, 0x55, 0x0a, 0x1f, 0xc3, 0xa6, 0x94, 0x9c, 0xde, 0xf8, 0xf6, 0xa9, 0xb8, 0x09, 0xb5, 0x0a,
|
||||
0xfe, 0xa7, 0x06, 0x83, 0xb2, 0xa1, 0xea, 0xe0, 0x7f, 0x9b, 0xff, 0xdf, 0xcd, 0x0e, 0xbd, 0x03,
|
||||
0x1d, 0x86, 0x5d, 0xcf, 0x0a, 0xc6, 0x63, 0x4a, 0xd8, 0x60, 0x69, 0xa8, 0x8d, 0x9a, 0x26, 0x70,
|
||||
0xd1, 0x0b, 0x21, 0x41, 0x7b, 0xd0, 0xb7, 0x65, 0x17, 0x5b, 0x11, 0xb9, 0x72, 0x29, 0x47, 0x5e,
|
||||
0x16, 0x81, 0xad, 0xda, 0x49, 0x77, 0x4b, 0x31, 0x32, 0xa0, 0xe7, 0x3a, 0xd7, 0x96, 0x18, 0x1e,
|
||||
0xe2, 0xea, 0xb7, 0x05, 0x5a, 0xc7, 0x75, 0xae, 0xbf, 0x71, 0x3d, 0x72, 0xca, 0x27, 0xc0, 0x53,
|
||||
0xd8, 0x98, 0x26, 0x7f, 0xe2, 0x3b, 0xe4, 0xba, 0x56, 0xd1, 0xbe, 0xcd, 0x16, 0x5b, 0x99, 0xa9,
|
||||
0x92, 0xed, 0x03, 0x72, 0xb9, 0x40, 0xfa, 0xb5, 0x03, 0x9f, 0x11, 0x9f, 0x09, 0x80, 0xae, 0xd9,
|
||||
0x17, 0x3b, 0xdc, 0xf9, 0x91, 0x94, 0x1b, 0xbf, 0x6a, 0x70, 0x77, 0x8a, 0x74, 0x8c, 0x19, 0xae,
|
||||
0xd5, 0x7a, 0x3a, 0xb4, 0xd3, 0xec, 0x17, 0xe5, 0x5e, 0xf2, 0xcd, 0xc7, 0xa2, 0xaa, 0x5e, 0x43,
|
||||
0xec, 0xa8, 0xaf, 0xaa, 0x01, 0xc8, 0x9d, 0xf8, 0x84, 0x38, 0x72, 0xba, 0xca, 0x63, 0x68, 0x4b,
|
||||
0xc1, 0x89, 0x63, 0x7c, 0x9e, 0xad, 0x8d, 0x0c, 0x4d, 0xe5, 0x78, 0x1f, 0xba, 0x15, 0xd9, 0x75,
|
||||
0xc6, 0x99, 0xc4, 0x3e, 0x02, 0x24, 0x8d, 0x9f, 0x07, 0xb1, 0x5f, 0x6f, 0xa6, 0xdc, 0x85, 0xb5,
|
||||
0x9c, 0x89, 0x6a, 0xec, 0x27, 0xb0, 0x2e, 0xc5, 0x2f, 0xfd, 0x49, 0x6d, 0xac, 0xcd, 0xa4, 0xac,
|
||||
0xa9, 0x91, 0x42, 0x3b, 0x4c, 0x9c, 0xe4, 0x1f, 0xb8, 0x5b, 0xc1, 0x36, 0x92, 0x08, 0xf2, 0x6f,
|
||||
0x9c, 0xf1, 0xbb, 0x06, 0x1b, 0xa6, 0x6a, 0x67, 0xf2, 0xdf, 0x0e, 0x8e, 0xec, 0xc5, 0x69, 0xcc,
|
||||
0xbc, 0x38, 0xcd, 0xe9, 0xc5, 0x19, 0x41, 0x9f, 0x06, 0x71, 0x64, 0x13, 0xcb, 0xc1, 0x0c, 0x5b,
|
||||
0x7e, 0xe0, 0x10, 0x75, 0xa0, 0x77, 0xa4, 0x9c, 0x1f, 0xe0, 0x77, 0x81, 0x43, 0x8c, 0x2d, 0xd8,
|
||||
0x2c, 0x05, 0xad, 0x12, 0xf2, 0x61, 0xf5, 0x28, 0x08, 0x6f, 0x78, 0x83, 0xd6, 0x4c, 0xa4, 0xe3,
|
||||
0x52, 0x2b, 0xb9, 0x64, 0x22, 0x93, 0xb6, 0xb9, 0xe2, 0xd2, 0x13, 0x79, 0xc3, 0xd4, 0xbe, 0x83,
|
||||
0x99, 0xdc, 0x6f, 0x24, 0xfb, 0xc7, 0x98, 0xf1, 0x7d, 0xe3, 0x29, 0xf4, 0xa7, 0xfe, 0xea, 0xf7,
|
||||
0xd6, 0x67, 0xb0, 0x6d, 0x12, 0xec, 0xc8, 0xe0, 0xc5, 0x55, 0xae, 0x3f, 0xee, 0xfe, 0xd0, 0xe0,
|
||||
0x5e, 0xb5, 0x71, 0x9d, 0x91, 0xc7, 0x2f, 0x77, 0x32, 0x52, 0x98, 0x3b, 0x21, 0x94, 0xe1, 0x49,
|
||||
0x28, 0xf2, 0x6e, 0x9a, 0x7d, 0x35, 0x57, 0xbe, 0x4f, 0xe4, 0xe5, 0x01, 0xd4, 0x28, 0x0d, 0x20,
|
||||
0x8e, 0x98, 0xd4, 0x27, 0x83, 0xd8, 0x94, 0x88, 0x8e, 0xac, 0x53, 0x0e, 0x31, 0xd5, 0x16, 0x88,
|
||||
0x2d, 0x89, 0xa8, 0x14, 0xc5, 0x48, 0xfb, 0x01, 0xe0, 0xd8, 0xa5, 0x97, 0x32, 0x2d, 0xde, 0x29,
|
||||
0x8e, 0x1b, 0xa9, 0xe7, 0x90, 0x2f, 0xb9, 0x04, 0x7b, 0x9e, 0x0a, 0x9a, 0x2f, 0xf9, 0x64, 0x88,
|
||||
0x29, 0x71, 0x54, 0x78, 0x62, 0xcd, 0x65, 0xe3, 0x88, 0x10, 0x15, 0x89, 0x58, 0x1b, 0xbf, 0x69,
|
||||
0xb0, 0xf2, 0x9c, 0x4c, 0x14, 0xf2, 0x0e, 0xc0, 0x79, 0x10, 0x05, 0x31, 0x73, 0x7d, 0x42, 0x85,
|
||||
0x83, 0x96, 0x99, 0x91, 0xfc, 0x73, 0x3f, 0x62, 0x52, 0x11, 0x6f, 0xac, 0x92, 0x13, 0x6b, 0x2e,
|
||||
0xbb, 0x20, 0x38, 0x54, 0x2f, 0x82, 0x58, 0x73, 0xa2, 0x47, 0x19, 0xb6, 0x2f, 0xc5, 0x03, 0xd0,
|
||||
0x34, 0xe5, 0xc7, 0xe1, 0xdb, 0x1e, 0x74, 0xd5, 0xdc, 0x12, 0x4c, 0x13, 0xbd, 0x86, 0x4e, 0x86,
|
||||
0xa1, 0xa2, 0xf7, 0xca, 0x44, 0xb4, 0xcc, 0x78, 0xf5, 0xf7, 0xe7, 0x68, 0xa9, 0x1b, 0xb3, 0x80,
|
||||
0x7c, 0xf8, 0x5f, 0x89, 0x01, 0xa2, 0x87, 0x65, 0xeb, 0x59, 0xfc, 0x52, 0x7f, 0x54, 0x4b, 0x37,
|
||||
0xf5, 0xc7, 0x60, 0xad, 0x82, 0xd2, 0xa1, 0xfd, 0x39, 0x28, 0x39, 0x5a, 0xa9, 0x3f, 0xae, 0xa9,
|
||||
0x9d, 0x7a, 0x7d, 0x03, 0xa8, 0xcc, 0xf7, 0xd0, 0xa3, 0xb9, 0x30, 0x53, 0x3e, 0xa9, 0xef, 0xd7,
|
||||
0x53, 0x9e, 0x99, 0xa8, 0x64, 0x82, 0x73, 0x13, 0xcd, 0x71, 0xcd, 0xb9, 0x89, 0x16, 0xe8, 0xe5,
|
||||
0x02, 0xba, 0x84, 0x7e, 0x91, 0x25, 0xa2, 0xbd, 0x59, 0x3f, 0x5d, 0x4a, 0x24, 0x54, 0x7f, 0x58,
|
||||
0x47, 0x35, 0x75, 0x66, 0x41, 0x37, 0xcb, 0xe5, 0x50, 0x45, 0xd3, 0x55, 0xb0, 0x52, 0x7d, 0x77,
|
||||
0x9e, 0x5a, 0x36, 0x9b, 0x22, 0xb7, 0xab, 0xca, 0x66, 0x06, 0x71, 0xac, 0xca, 0x66, 0x16, 0x55,
|
||||
0x34, 0x16, 0xd0, 0x4f, 0xb0, 0x5a, 0x20, 0x45, 0x68, 0x74, 0x1b, 0x40, 0x96, 0x6e, 0xe9, 0x7b,
|
||||
0x35, 0x34, 0x13, 0x4f, 0x1f, 0x6a, 0xe8, 0x1c, 0xee, 0xe4, 0xb9, 0x09, 0x7a, 0x70, 0x1b, 0x40,
|
||||
0x86, 0x58, 0xe9, 0xa3, 0xf9, 0x8a, 0x19, 0x47, 0xaf, 0xa1, 0x93, 0x21, 0x25, 0x55, 0xc3, 0xa3,
|
||||
0x4c, 0x73, 0xaa, 0x86, 0x47, 0x15, 0xb3, 0x59, 0x40, 0x67, 0xd0, 0xcb, 0xd1, 0x14, 0xb4, 0x3b,
|
||||
0xcb, 0x32, 0x4f, 0x7e, 0xf4, 0x07, 0x73, 0xf5, 0xb2, 0x4d, 0x96, 0x65, 0x2f, 0x68, 0x66, 0x70,
|
||||
0xf9, 0x01, 0xb8, 0x3b, 0x4f, 0x2d, 0x75, 0x70, 0x01, 0xab, 0x05, 0x42, 0x51, 0x75, 0xee, 0xd5,
|
||||
0x44, 0xa9, 0xea, 0xdc, 0x67, 0xb1, 0x93, 0x05, 0xf4, 0x33, 0xac, 0x57, 0xbd, 0xdd, 0xe8, 0x71,
|
||||
0x15, 0xc8, 0x4c, 0x82, 0xa0, 0x1f, 0xd4, 0x55, 0x4f, 0x1d, 0xbf, 0x84, 0x76, 0x42, 0x54, 0xd0,
|
||||
0xfd, 0xb2, 0x75, 0x81, 0x34, 0xe9, 0xc6, 0x6d, 0x2a, 0xd3, 0xe6, 0x3a, 0x5b, 0x12, 0xff, 0xc2,
|
||||
0x3c, 0xf9, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x7d, 0x0d, 0xbb, 0x9c, 0x11, 0x00, 0x00,
|
||||
// 1169 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6f, 0xdc, 0x44,
|
||||
0x10, 0x8f, 0x7b, 0x77, 0xf9, 0x98, 0xbb, 0xd0, 0x63, 0x93, 0x26, 0x57, 0xb7, 0x84, 0xeb, 0x42,
|
||||
0xcb, 0xa5, 0x4d, 0x03, 0xa4, 0x02, 0x5a, 0x9e, 0x80, 0x84, 0x8a, 0x3c, 0x94, 0x4a, 0x0e, 0xad,
|
||||
0x90, 0x40, 0xb2, 0x36, 0xf6, 0x26, 0xb1, 0xe2, 0xf3, 0xba, 0xde, 0x75, 0xda, 0xf0, 0xef, 0xf0,
|
||||
0xc2, 0x3b, 0x12, 0xff, 0x0e, 0x7f, 0x08, 0x2f, 0x68, 0x3f, 0xec, 0xf8, 0xeb, 0x7a, 0xe6, 0xe3,
|
||||
0x6d, 0x3d, 0x3b, 0xf3, 0x9b, 0x99, 0xdd, 0x99, 0xd9, 0x9f, 0x0c, 0x6b, 0x17, 0x2c, 0x4c, 0xa7,
|
||||
0xd4, 0xe5, 0x34, 0xb9, 0xa0, 0xc9, 0x6e, 0x9c, 0x30, 0xc1, 0xd0, 0xb0, 0x24, 0x74, 0xe3, 0x63,
|
||||
0xfc, 0x31, 0xa0, 0x6f, 0x88, 0xf0, 0xce, 0x0e, 0x68, 0x48, 0x05, 0x75, 0xe8, 0xab, 0x94, 0x72,
|
||||
0x81, 0x6e, 0xc2, 0xf2, 0x49, 0x10, 0x52, 0x37, 0xf0, 0xf9, 0xc8, 0x1a, 0x77, 0x26, 0x2b, 0xce,
|
||||
0x92, 0xfc, 0x3e, 0xf4, 0x39, 0x7e, 0x0e, 0x6b, 0x25, 0x03, 0x1e, 0xb3, 0x88, 0x53, 0xf4, 0x18,
|
||||
0x96, 0x12, 0xca, 0xd3, 0x50, 0x68, 0x83, 0xfe, 0xde, 0xd6, 0x6e, 0xd5, 0xd7, 0x6e, 0x6e, 0x92,
|
||||
0x86, 0xc2, 0xc9, 0xd4, 0x71, 0x00, 0x83, 0xe2, 0x06, 0xda, 0x84, 0x25, 0xe3, 0x7b, 0x64, 0x8d,
|
||||
0xad, 0xc9, 0x8a, 0xb3, 0xa8, 0x5d, 0xa3, 0x0d, 0x58, 0xe4, 0x82, 0x88, 0x94, 0x8f, 0xae, 0x8d,
|
||||
0xad, 0x49, 0xcf, 0x31, 0x5f, 0x68, 0x1d, 0x7a, 0x34, 0x49, 0x58, 0x32, 0xea, 0x28, 0x75, 0xfd,
|
||||
0x81, 0x10, 0x74, 0x79, 0xf0, 0x0b, 0x1d, 0x75, 0xc7, 0xd6, 0x64, 0xd5, 0x51, 0x6b, 0xbc, 0x04,
|
||||
0xbd, 0x6f, 0xa7, 0xb1, 0xb8, 0xc4, 0x5f, 0xc0, 0xe8, 0x25, 0xf1, 0xd2, 0x74, 0xfa, 0x52, 0xc5,
|
||||
0xb8, 0x7f, 0x46, 0xbd, 0xf3, 0x2c, 0xf7, 0x5b, 0xb0, 0x62, 0x22, 0x37, 0x11, 0xac, 0x3a, 0xcb,
|
||||
0x5a, 0x70, 0xe8, 0xe3, 0xaf, 0xe0, 0x66, 0x83, 0xa1, 0x39, 0x83, 0x0f, 0x60, 0xf5, 0x94, 0x24,
|
||||
0xc7, 0xe4, 0x94, 0xba, 0x09, 0x11, 0x01, 0x53, 0xd6, 0x96, 0x33, 0x30, 0x42, 0x47, 0xca, 0xf0,
|
||||
0x4f, 0x60, 0x97, 0x10, 0xd8, 0x34, 0x26, 0x9e, 0x68, 0xe3, 0x1c, 0x8d, 0xa1, 0x1f, 0x27, 0x94,
|
||||
0x84, 0x21, 0xf3, 0x88, 0xa0, 0xea, 0x14, 0x3a, 0x4e, 0x51, 0x84, 0xdf, 0x83, 0x5b, 0x8d, 0xe0,
|
||||
0x3a, 0x40, 0xfc, 0xb8, 0x12, 0x3d, 0x9b, 0x4e, 0x83, 0x56, 0xae, 0xf1, 0xed, 0x5a, 0xd4, 0xca,
|
||||
0xd2, 0xe0, 0x3e, 0xa9, 0xec, 0x86, 0x94, 0x44, 0x69, 0xdc, 0x0a, 0xb8, 0x1a, 0x71, 0x66, 0x9a,
|
||||
0x23, 0x6f, 0xea, 0xe2, 0xd8, 0x67, 0x61, 0x48, 0x3d, 0x11, 0xb0, 0x28, 0x83, 0xdd, 0x02, 0xf0,
|
||||
0x72, 0xa1, 0x29, 0x95, 0x82, 0x04, 0xdb, 0x30, 0xaa, 0x9b, 0x1a, 0xd8, 0xdf, 0x2c, 0x58, 0xfb,
|
||||
0x9a, 0xf3, 0xe0, 0x34, 0xd2, 0x6e, 0x5b, 0x1d, 0x7f, 0xd9, 0xe1, 0xb5, 0xaa, 0xc3, 0xea, 0xf5,
|
||||
0x74, 0x6a, 0xd7, 0x23, 0x35, 0x12, 0x1a, 0x87, 0x81, 0x47, 0x14, 0x44, 0x57, 0x41, 0x14, 0x45,
|
||||
0x68, 0x08, 0x1d, 0x21, 0xc2, 0x51, 0x4f, 0xed, 0xc8, 0x25, 0xde, 0x80, 0xf5, 0x72, 0xa4, 0x26,
|
||||
0x85, 0xcf, 0x61, 0x53, 0x4b, 0x8e, 0x2e, 0x23, 0xef, 0x48, 0x75, 0x42, 0xab, 0x03, 0xff, 0xcb,
|
||||
0x82, 0x51, 0xdd, 0xd0, 0x54, 0xf0, 0x7f, 0xcd, 0xff, 0x9f, 0x66, 0x87, 0xde, 0x87, 0xbe, 0x20,
|
||||
0x41, 0xe8, 0xb2, 0x93, 0x13, 0x4e, 0xc5, 0x68, 0x71, 0x6c, 0x4d, 0xba, 0x0e, 0x48, 0xd1, 0x73,
|
||||
0x25, 0x41, 0xdb, 0x30, 0xf4, 0x74, 0x15, 0xbb, 0x09, 0xbd, 0x08, 0xb8, 0x44, 0x5e, 0x52, 0x81,
|
||||
0x5d, 0xf7, 0xb2, 0xea, 0xd6, 0x62, 0x84, 0x61, 0x35, 0xf0, 0xdf, 0xb8, 0x6a, 0x78, 0xa8, 0xd6,
|
||||
0x5f, 0x56, 0x68, 0xfd, 0xc0, 0x7f, 0xf3, 0x34, 0x08, 0xe9, 0x91, 0x9c, 0x00, 0xdf, 0xc1, 0x9a,
|
||||
0x4e, 0xfe, 0x29, 0x0b, 0x43, 0xf6, 0xba, 0xd5, 0xbd, 0xaf, 0x43, 0x8f, 0x07, 0x91, 0xa7, 0x1b,
|
||||
0xae, 0xeb, 0xe8, 0x0f, 0xfc, 0x04, 0xd6, 0xcb, 0x48, 0xe6, 0x08, 0xef, 0xc0, 0x40, 0x45, 0xe0,
|
||||
0xb1, 0x48, 0xd0, 0x48, 0x28, 0xb4, 0x81, 0xd3, 0x97, 0xb2, 0x7d, 0x2d, 0xc2, 0x9f, 0x02, 0xd2,
|
||||
0xa6, 0xcf, 0x58, 0x1a, 0xb5, 0xeb, 0xbf, 0x1b, 0x59, 0xdc, 0xc6, 0xc4, 0x14, 0xc1, 0xa3, 0x2c,
|
||||
0x88, 0x17, 0xd1, 0xb4, 0x35, 0xd6, 0x26, 0xdc, 0xa8, 0x18, 0x19, 0xb4, 0xbd, 0xcc, 0x49, 0xf9,
|
||||
0x31, 0x78, 0x2b, 0xd8, 0x46, 0x16, 0x41, 0xf9, 0x3d, 0xc0, 0xbf, 0x5b, 0xb0, 0xe1, 0x98, 0xab,
|
||||
0xa7, 0xff, 0x6f, 0x93, 0x15, 0x8b, 0xac, 0x33, 0xb3, 0xc8, 0xba, 0x57, 0x45, 0x36, 0x81, 0x21,
|
||||
0x67, 0x69, 0xe2, 0x51, 0xd7, 0x27, 0x82, 0xb8, 0x11, 0xf3, 0xa9, 0xa9, 0xc1, 0x77, 0xb4, 0xfc,
|
||||
0x80, 0x08, 0xf2, 0x3d, 0xf3, 0x29, 0xbe, 0x09, 0x9b, 0xb5, 0xa0, 0x4d, 0x42, 0x11, 0x5c, 0xdf,
|
||||
0x67, 0xf1, 0xa5, 0xac, 0xa4, 0x96, 0x89, 0xf4, 0x03, 0xee, 0x66, 0x05, 0xa9, 0x32, 0x59, 0x76,
|
||||
0x56, 0x02, 0x7e, 0xa8, 0xab, 0xd1, 0xec, 0xfb, 0x44, 0xe8, 0xfd, 0x4e, 0xb6, 0x7f, 0x40, 0x84,
|
||||
0xdc, 0xc7, 0x9f, 0xc1, 0xf0, 0xca, 0x5f, 0xfb, 0xda, 0xfa, 0x12, 0x6e, 0x39, 0x94, 0xf8, 0xa6,
|
||||
0x34, 0x65, 0xd9, 0xb7, 0x1f, 0x0d, 0x7f, 0x5a, 0x70, 0xbb, 0xd9, 0xb8, 0xcd, 0x78, 0xd8, 0x01,
|
||||
0x94, 0xb7, 0x9f, 0x08, 0xa6, 0x94, 0x0b, 0x32, 0x8d, 0x4d, 0xcf, 0x0c, 0x4d, 0x0f, 0xfe, 0x90,
|
||||
0xc9, 0xeb, 0xcd, 0xda, 0xa9, 0x35, 0xab, 0x44, 0xcc, 0xce, 0xa7, 0x80, 0xd8, 0xd5, 0x88, 0xbe,
|
||||
0x3e, 0xa7, 0x12, 0x62, 0xae, 0xad, 0x10, 0x7b, 0x1a, 0xd1, 0x28, 0xaa, 0xf6, 0xff, 0x11, 0xe0,
|
||||
0x20, 0xe0, 0xe7, 0x3a, 0x2d, 0x59, 0x29, 0x7e, 0x90, 0x98, 0xa7, 0x43, 0x2e, 0xa5, 0x84, 0x84,
|
||||
0xa1, 0x09, 0x5a, 0x2e, 0x25, 0x8d, 0x48, 0x39, 0xf5, 0x4d, 0x78, 0x6a, 0x2d, 0x65, 0x27, 0x09,
|
||||
0xa5, 0x26, 0x12, 0xb5, 0xc6, 0xbf, 0x5a, 0xb0, 0xf2, 0x8c, 0x4e, 0x0d, 0xf2, 0x16, 0xc0, 0x29,
|
||||
0x4b, 0x58, 0x2a, 0x82, 0x88, 0x72, 0xe5, 0xa0, 0xe7, 0x14, 0x24, 0xff, 0xde, 0x8f, 0xa2, 0x35,
|
||||
0x34, 0x3c, 0x31, 0xc9, 0xa9, 0xb5, 0x94, 0x9d, 0x51, 0x12, 0x9b, 0xe9, 0xa9, 0xd6, 0x6a, 0x68,
|
||||
0x09, 0xe2, 0x9d, 0xab, 0x61, 0x29, 0x87, 0x96, 0xfc, 0xd8, 0xfb, 0x63, 0x00, 0x03, 0x33, 0xfc,
|
||||
0x15, 0x2b, 0x43, 0x3f, 0x43, 0xbf, 0xc0, 0xe6, 0xd0, 0x87, 0x75, 0xd2, 0x56, 0x67, 0x87, 0xf6,
|
||||
0xdd, 0x39, 0x5a, 0xa6, 0x63, 0x16, 0x50, 0x04, 0xef, 0xd6, 0xd8, 0x12, 0xba, 0x5f, 0xb7, 0x9e,
|
||||
0xc5, 0xc5, 0xec, 0x07, 0xad, 0x74, 0x73, 0x7f, 0x02, 0xd6, 0x1a, 0xe8, 0x0f, 0xda, 0x99, 0x83,
|
||||
0x52, 0xa2, 0x60, 0xf6, 0xc3, 0x96, 0xda, 0xb9, 0xd7, 0x57, 0x80, 0xea, 0xdc, 0x08, 0x3d, 0x98,
|
||||
0x0b, 0x73, 0xc5, 0xbd, 0xec, 0x9d, 0x76, 0xca, 0x33, 0x13, 0xd5, 0xac, 0x69, 0x6e, 0xa2, 0x25,
|
||||
0x5e, 0x36, 0x37, 0xd1, 0x0a, 0x15, 0x5b, 0x40, 0xe7, 0x30, 0xac, 0x32, 0x2a, 0xb4, 0x3d, 0x8b,
|
||||
0xe6, 0xd7, 0x08, 0x9b, 0x7d, 0xbf, 0x8d, 0x6a, 0xee, 0xcc, 0x85, 0x41, 0x91, 0xf7, 0xa0, 0x86,
|
||||
0xa2, 0x6b, 0x60, 0x70, 0xf6, 0xbd, 0x79, 0x6a, 0xc5, 0x6c, 0xaa, 0x3c, 0xa8, 0x29, 0x9b, 0x19,
|
||||
0x24, 0xab, 0x29, 0x9b, 0x59, 0xb4, 0x0a, 0x2f, 0x20, 0x92, 0xf5, 0x9d, 0x66, 0x0b, 0x4d, 0xd9,
|
||||
0x34, 0xf0, 0x92, 0xa6, 0x6c, 0x9a, 0x48, 0x07, 0x5e, 0xf8, 0xc4, 0x92, 0xad, 0x5c, 0xa0, 0x08,
|
||||
0x4d, 0xad, 0x5c, 0x27, 0x1d, 0xf6, 0xdd, 0x39, 0x5a, 0x79, 0x02, 0xc7, 0xb0, 0x5a, 0x22, 0x0d,
|
||||
0x68, 0x66, 0x68, 0x65, 0x2a, 0x62, 0x7f, 0x34, 0x57, 0xaf, 0x78, 0xe5, 0x45, 0x2e, 0x31, 0xfb,
|
||||
0x90, 0xca, 0xe3, 0xe8, 0xde, 0x3c, 0xb5, 0xdc, 0xc1, 0x19, 0x5c, 0xaf, 0x3c, 0xef, 0x68, 0x52,
|
||||
0x37, 0x6e, 0xa6, 0x2d, 0xf6, 0x76, 0x0b, 0xcd, 0xdc, 0xd3, 0x6b, 0x58, 0x6f, 0x7a, 0x49, 0xd1,
|
||||
0xc3, 0x26, 0x90, 0x99, 0xcf, 0xb5, 0xbd, 0xdb, 0x56, 0x3d, 0x77, 0xfc, 0x02, 0x96, 0x33, 0xda,
|
||||
0x80, 0xee, 0xd4, 0xad, 0x2b, 0x14, 0xc6, 0xc6, 0x6f, 0x53, 0xb9, 0x2a, 0xae, 0xe3, 0x45, 0xf5,
|
||||
0xff, 0xe0, 0xd1, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x13, 0x86, 0xf1, 0x0f, 0x56, 0x10, 0x00,
|
||||
0x00,
|
||||
}
|
||||
|
|
|
|||
67
weed/server/volume_grpc_follow.go
Normal file
67
weed/server/volume_grpc_follow.go
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
)
|
||||
|
||||
func (vs *VolumeServer) VolumeFollow(req *volume_server_pb.VolumeFollowRequest, stream volume_server_pb.VolumeServer_VolumeFollowServer) error {
|
||||
|
||||
v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
return fmt.Errorf("not found volume id %d", req.VolumeId)
|
||||
}
|
||||
|
||||
stopOffset := v.Size()
|
||||
foundOffset, isLastOne, err := v.BinarySearchByAppendAtNs(req.Since)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail to locate by appendAtNs %d: %s", req.Since, err)
|
||||
}
|
||||
|
||||
if isLastOne {
|
||||
return nil
|
||||
}
|
||||
|
||||
startOffset := int64(foundOffset) * int64(types.NeedlePaddingSize)
|
||||
|
||||
buf := make([]byte, 1024*1024*2)
|
||||
return sendFileContent(v.DataFile(), buf, startOffset, stopOffset, stream)
|
||||
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server_pb.VolumeSyncStatusRequest) (*volume_server_pb.VolumeSyncStatusResponse, error) {
|
||||
|
||||
v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
|
||||
}
|
||||
|
||||
resp := v.GetVolumeSyncStatus()
|
||||
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
||||
func sendFileContent(datFile *os.File, buf []byte, startOffset, stopOffset int64, stream volume_server_pb.VolumeServer_VolumeFollowServer) error {
|
||||
var blockSizeLimit = int64(len(buf))
|
||||
for i := int64(0); i < stopOffset-startOffset; i += blockSizeLimit {
|
||||
n, readErr := datFile.ReadAt(buf, startOffset+i)
|
||||
if readErr == nil || readErr == io.EOF {
|
||||
resp := &volume_server_pb.VolumeFollowResponse{}
|
||||
resp.FileContent = buf[:int64(n)]
|
||||
sendErr := stream.Send(resp)
|
||||
if sendErr != nil {
|
||||
return sendErr
|
||||
}
|
||||
} else {
|
||||
return readErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -113,7 +113,7 @@ func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_se
|
|||
return resp, nil
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) (error) {
|
||||
func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {
|
||||
|
||||
v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
|
|
|
|||
|
|
@ -1,101 +0,0 @@
|
|||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server_pb.VolumeSyncStatusRequest) (*volume_server_pb.VolumeSyncStatusResponse, error) {
|
||||
|
||||
v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
|
||||
}
|
||||
|
||||
resp := v.GetVolumeSyncStatus()
|
||||
|
||||
glog.V(2).Infof("volume sync status %d", req.VolumeId)
|
||||
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) VolumeSyncIndex(req *volume_server_pb.VolumeSyncIndexRequest, stream volume_server_pb.VolumeServer_VolumeSyncIndexServer) error {
|
||||
|
||||
v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
return fmt.Errorf("not found volume id %d", req.VolumeId)
|
||||
}
|
||||
|
||||
content, err := v.IndexFileContent()
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("sync volume %d index: %v", req.VolumeId, err)
|
||||
} else {
|
||||
glog.V(2).Infof("sync volume %d index", req.VolumeId)
|
||||
}
|
||||
|
||||
const blockSizeLimit = 1024 * 1024 * 2
|
||||
for i := 0; i < len(content); i += blockSizeLimit {
|
||||
blockSize := len(content) - i
|
||||
if blockSize > blockSizeLimit {
|
||||
blockSize = blockSizeLimit
|
||||
}
|
||||
resp := &volume_server_pb.VolumeSyncIndexResponse{}
|
||||
resp.IndexFileContent = content[i : i+blockSize]
|
||||
stream.Send(resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (vs *VolumeServer) VolumeSyncData(req *volume_server_pb.VolumeSyncDataRequest, stream volume_server_pb.VolumeServer_VolumeSyncDataServer) error {
|
||||
|
||||
v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
|
||||
if v == nil {
|
||||
return fmt.Errorf("not found volume id %d", req.VolumeId)
|
||||
}
|
||||
|
||||
if uint32(v.SuperBlock.CompactRevision) != req.Revision {
|
||||
return fmt.Errorf("requested volume revision is %d, but current revision is %d", req.Revision, v.SuperBlock.CompactRevision)
|
||||
}
|
||||
|
||||
content, err := storage.ReadNeedleBlob(v.DataFile(), int64(req.Offset)*types.NeedlePaddingSize, req.Size, v.Version())
|
||||
if err != nil {
|
||||
return fmt.Errorf("read offset:%d size:%d", req.Offset, req.Size)
|
||||
}
|
||||
|
||||
id, err := types.ParseNeedleId(req.NeedleId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing needle id %s: %v", req.NeedleId, err)
|
||||
}
|
||||
n := new(storage.Needle)
|
||||
n.ParseNeedleHeader(content)
|
||||
if id != n.Id {
|
||||
return fmt.Errorf("expected file entry id %d, but found %d", id, n.Id)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("sync volume %d data: %v", req.VolumeId, err)
|
||||
}
|
||||
|
||||
const blockSizeLimit = 1024 * 1024 * 2
|
||||
for i := 0; i < len(content); i += blockSizeLimit {
|
||||
blockSize := len(content) - i
|
||||
if blockSize > blockSizeLimit {
|
||||
blockSize = blockSizeLimit
|
||||
}
|
||||
resp := &volume_server_pb.VolumeSyncDataResponse{}
|
||||
resp.FileContent = content[i : i+blockSize]
|
||||
stream.Send(resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
|
@ -42,10 +42,10 @@ func (c *commandFsDu) Do(args []string, commandEnv *commandEnv, writer io.Writer
|
|||
|
||||
dir, name := filer2.FullPath(path).DirAndName()
|
||||
if strings.HasSuffix(path, "/") {
|
||||
if path == "/"{
|
||||
if path == "/" {
|
||||
dir, name = "/", ""
|
||||
}else{
|
||||
dir, name = path[0 : len(path)-1], ""
|
||||
} else {
|
||||
dir, name = path[0:len(path)-1], ""
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ func (s *Store) CompactVolume(vid VolumeId, preallocate int64) error {
|
|||
}
|
||||
func (s *Store) CommitCompactVolume(vid VolumeId) error {
|
||||
if v := s.findVolume(vid); v != nil {
|
||||
return v.commitCompact()
|
||||
return v.CommitCompact()
|
||||
}
|
||||
return fmt.Errorf("volume id %d is not found during commit compact", vid)
|
||||
}
|
||||
|
|
|
|||
253
weed/storage/volume_follow.go
Normal file
253
weed/storage/volume_follow.go
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
func (v *Volume) GetVolumeSyncStatus() *volume_server_pb.VolumeSyncStatusResponse {
|
||||
var syncStatus = &volume_server_pb.VolumeSyncStatusResponse{}
|
||||
if stat, err := v.dataFile.Stat(); err == nil {
|
||||
syncStatus.TailOffset = uint64(stat.Size())
|
||||
}
|
||||
syncStatus.Collection = v.Collection
|
||||
syncStatus.IdxFileSize = v.nm.IndexFileSize()
|
||||
syncStatus.CompactRevision = uint32(v.SuperBlock.CompactRevision)
|
||||
syncStatus.Ttl = v.SuperBlock.Ttl.String()
|
||||
syncStatus.Replication = v.SuperBlock.ReplicaPlacement.String()
|
||||
return syncStatus
|
||||
}
|
||||
|
||||
// The volume sync with a master volume via 2 steps:
|
||||
// 1. The slave checks master side to find subscription checkpoint
|
||||
// to setup the replication.
|
||||
// 2. The slave receives the updates from master
|
||||
|
||||
/*
|
||||
Assume the slave volume needs to follow the master volume.
|
||||
|
||||
The master volume could be compacted, and could be many files ahead of
|
||||
slave volume.
|
||||
|
||||
Step 0: // implemented in command/backup.go, to avoid dat file size overflow.
|
||||
0.1 If slave compact version is less than the master, do a local compaction, and set
|
||||
local compact version the same as the master.
|
||||
0.2 If the slave size is still bigger than the master, discard local copy and do a full copy.
|
||||
|
||||
Step 1:
|
||||
The slave volume ask the master by the last modification time t.
|
||||
The master do a binary search in volume (use .idx as an array, and check the appendAtNs in .dat file),
|
||||
to find the first entry with appendAtNs > t.
|
||||
|
||||
Step 2:
|
||||
The master send content bytes to the slave. The bytes are not chunked by needle.
|
||||
|
||||
Step 3:
|
||||
The slave generate the needle map for the new bytes. (This may be optimized to incrementally
|
||||
update needle map when receiving new .dat bytes. But seems not necessary now.)
|
||||
|
||||
*/
|
||||
|
||||
func (v *Volume) Follow(volumeServer string, grpcDialOption grpc.DialOption) error {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
startFromOffset := v.Size()
|
||||
appendAtNs, err := v.findLastAppendAtNs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
|
||||
stream, err := client.VolumeFollow(ctx, &volume_server_pb.VolumeFollowRequest{
|
||||
VolumeId: uint32(v.Id),
|
||||
Since: appendAtNs,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v.dataFile.Seek(startFromOffset, io.SeekStart)
|
||||
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
} else {
|
||||
return recvErr
|
||||
}
|
||||
}
|
||||
|
||||
_, writeErr := v.dataFile.Write(resp.FileContent)
|
||||
if writeErr != nil {
|
||||
return writeErr
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// add to needle map
|
||||
return ScanVolumeFileFrom(v.version, v.dataFile, startFromOffset, &VolumeFileScanner4GenIdx{v: v})
|
||||
|
||||
}
|
||||
|
||||
func (v *Volume) findLastAppendAtNs() (uint64, error) {
|
||||
offset, err := v.locateLastAppendEntry()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if offset == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return v.readAppendAtNs(offset)
|
||||
}
|
||||
|
||||
func (v *Volume) locateLastAppendEntry() (Offset, error) {
|
||||
indexFile, e := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644)
|
||||
if e != nil {
|
||||
return 0, fmt.Errorf("cannot read %s.idx: %v", v.FileName(), e)
|
||||
}
|
||||
defer indexFile.Close()
|
||||
|
||||
fi, err := indexFile.Stat()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("file %s stat error: %v", indexFile.Name(), err)
|
||||
}
|
||||
fileSize := fi.Size()
|
||||
if fileSize%NeedleEntrySize != 0 {
|
||||
return 0, fmt.Errorf("unexpected file %s size: %d", indexFile.Name(), fileSize)
|
||||
}
|
||||
if fileSize == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
bytes := make([]byte, NeedleEntrySize)
|
||||
n, e := indexFile.ReadAt(bytes, fileSize-NeedleEntrySize)
|
||||
if n != NeedleEntrySize {
|
||||
return 0, fmt.Errorf("file %s read error: %v", indexFile.Name(), e)
|
||||
}
|
||||
_, offset, _ := IdxFileEntry(bytes)
|
||||
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) {
|
||||
|
||||
n, bodyLength, err := ReadNeedleHeader(v.dataFile, v.SuperBlock.version, int64(offset)*NeedlePaddingSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ReadNeedleHeader: %v", err)
|
||||
}
|
||||
err = n.ReadNeedleBody(v.dataFile, v.SuperBlock.version, int64(offset)*NeedlePaddingSize+int64(NeedleEntrySize), bodyLength)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ReadNeedleBody offset %d, bodyLength %d: %v", int64(offset)*NeedlePaddingSize, bodyLength, err)
|
||||
}
|
||||
return n.AppendAtNs, nil
|
||||
|
||||
}
|
||||
|
||||
// on server side
|
||||
func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast bool, err error) {
|
||||
indexFile, openErr := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644)
|
||||
if openErr != nil {
|
||||
err = fmt.Errorf("cannot read %s.idx: %v", v.FileName(), openErr)
|
||||
return
|
||||
}
|
||||
defer indexFile.Close()
|
||||
|
||||
fi, statErr := indexFile.Stat()
|
||||
if statErr != nil {
|
||||
err = fmt.Errorf("file %s stat error: %v", indexFile.Name(), statErr)
|
||||
return
|
||||
}
|
||||
fileSize := fi.Size()
|
||||
if fileSize%NeedleEntrySize != 0 {
|
||||
err = fmt.Errorf("unexpected file %s size: %d", indexFile.Name(), fileSize)
|
||||
return
|
||||
}
|
||||
|
||||
bytes := make([]byte, NeedleEntrySize)
|
||||
entryCount := fileSize / NeedleEntrySize
|
||||
l := int64(0)
|
||||
h := entryCount
|
||||
|
||||
for l < h {
|
||||
|
||||
m := (l + h) / 2
|
||||
|
||||
if m == entryCount {
|
||||
return 0, true, nil
|
||||
}
|
||||
|
||||
// read the appendAtNs for entry m
|
||||
offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, m)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
mNs, nsReadErr := v.readAppendAtNs(offset)
|
||||
if nsReadErr != nil {
|
||||
err = nsReadErr
|
||||
return
|
||||
}
|
||||
|
||||
// move the boundary
|
||||
if mNs <= sinceNs {
|
||||
l = m + 1
|
||||
} else {
|
||||
h = m
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if l == entryCount {
|
||||
return 0, true, nil
|
||||
}
|
||||
|
||||
offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, l)
|
||||
|
||||
return offset, false, err
|
||||
|
||||
}
|
||||
|
||||
// bytes is of size NeedleEntrySize
|
||||
func (v *Volume) readAppendAtNsForIndexEntry(indexFile *os.File, bytes []byte, m int64) (Offset, error) {
|
||||
if _, readErr := indexFile.ReadAt(bytes, m*NeedleEntrySize); readErr != nil && readErr != io.EOF {
|
||||
return 0, readErr
|
||||
}
|
||||
_, offset, _ := IdxFileEntry(bytes)
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
// generate the volume idx
|
||||
type VolumeFileScanner4GenIdx struct {
|
||||
v *Volume
|
||||
}
|
||||
|
||||
func (scanner *VolumeFileScanner4GenIdx) VisitSuperBlock(superBlock SuperBlock) error {
|
||||
return nil
|
||||
|
||||
}
|
||||
func (scanner *VolumeFileScanner4GenIdx) ReadNeedleBody() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (scanner *VolumeFileScanner4GenIdx) VisitNeedle(n *Needle, offset int64) error {
|
||||
if n.Size > 0 && n.Size != TombstoneFileSize {
|
||||
return scanner.v.nm.Put(n.Id, Offset(offset/NeedlePaddingSize), n.Size)
|
||||
}
|
||||
return scanner.v.nm.Delete(n.Id, Offset(offset/NeedlePaddingSize))
|
||||
}
|
||||
39
weed/storage/volume_follow_test.go
Normal file
39
weed/storage/volume_follow_test.go
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBinarySearch(t *testing.T) {
|
||||
var testInput []int
|
||||
testInput = []int{-1, 0, 3, 5, 9, 12}
|
||||
|
||||
if 3 != binarySearchForLargerThanTarget(testInput, 4) {
|
||||
t.Errorf("failed to find target %d", 4)
|
||||
}
|
||||
if 3 != binarySearchForLargerThanTarget(testInput, 3) {
|
||||
t.Errorf("failed to find target %d", 3)
|
||||
}
|
||||
if 6 != binarySearchForLargerThanTarget(testInput, 12) {
|
||||
t.Errorf("failed to find target %d", 12)
|
||||
}
|
||||
if 1 != binarySearchForLargerThanTarget(testInput, -1) {
|
||||
t.Errorf("failed to find target %d", -1)
|
||||
}
|
||||
if 0 != binarySearchForLargerThanTarget(testInput, -2) {
|
||||
t.Errorf("failed to find target %d", -2)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func binarySearchForLargerThanTarget(nums []int, target int) int {
|
||||
l := 0
|
||||
h := len(nums)
|
||||
for l < h {
|
||||
m := (l + h) / 2
|
||||
if nums[m] <= target {
|
||||
l = m + 1
|
||||
} else {
|
||||
h = m
|
||||
}
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
|
@ -180,30 +180,37 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
|
|||
volumeFileScanner VolumeFileScanner) (err error) {
|
||||
var v *Volume
|
||||
if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil {
|
||||
return fmt.Errorf("Failed to load volume %d: %v", id, err)
|
||||
return fmt.Errorf("failed to load volume %d: %v", id, err)
|
||||
}
|
||||
if err = volumeFileScanner.VisitSuperBlock(v.SuperBlock); err != nil {
|
||||
return fmt.Errorf("Failed to process volume %d super block: %v", id, err)
|
||||
return fmt.Errorf("failed to process volume %d super block: %v", id, err)
|
||||
}
|
||||
defer v.Close()
|
||||
|
||||
version := v.Version()
|
||||
|
||||
offset := int64(v.SuperBlock.BlockSize())
|
||||
n, rest, e := ReadNeedleHeader(v.dataFile, version, offset)
|
||||
|
||||
return ScanVolumeFileFrom(version, v.dataFile, offset, volumeFileScanner)
|
||||
}
|
||||
|
||||
func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
|
||||
n, rest, e := ReadNeedleHeader(dataFile, version, offset)
|
||||
if e != nil {
|
||||
err = fmt.Errorf("cannot read needle header: %v", e)
|
||||
return
|
||||
if e == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("cannot read %s at offset %d: %v", dataFile.Name(), offset, e)
|
||||
}
|
||||
for n != nil {
|
||||
if volumeFileScanner.ReadNeedleBody() {
|
||||
if err = n.ReadNeedleBody(v.dataFile, version, offset+NeedleEntrySize, rest); err != nil {
|
||||
if err = n.ReadNeedleBody(dataFile, version, offset+NeedleEntrySize, rest); err != nil {
|
||||
glog.V(0).Infof("cannot read needle body: %v", err)
|
||||
//err = fmt.Errorf("cannot read needle body: %v", err)
|
||||
//return
|
||||
}
|
||||
}
|
||||
err = volumeFileScanner.VisitNeedle(n, offset)
|
||||
err := volumeFileScanner.VisitNeedle(n, offset)
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -212,14 +219,13 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
|
|||
}
|
||||
offset += NeedleEntrySize + rest
|
||||
glog.V(4).Infof("==> new entry offset %d", offset)
|
||||
if n, rest, err = ReadNeedleHeader(v.dataFile, version, offset); err != nil {
|
||||
if n, rest, err = ReadNeedleHeader(dataFile, version, offset); err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("cannot read needle header: %v", err)
|
||||
return fmt.Errorf("cannot read needle header at offset %d: %v", offset, err)
|
||||
}
|
||||
glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
|
||||
}
|
||||
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,225 +0,0 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
// The volume sync with a master volume via 2 steps:
|
||||
// 1. The slave checks master side to find subscription checkpoint
|
||||
// to setup the replication.
|
||||
// 2. The slave receives the updates from master
|
||||
|
||||
/*
|
||||
Assume the slave volume needs to follow the master volume.
|
||||
|
||||
The master volume could be compacted, and could be many files ahead of
|
||||
slave volume.
|
||||
|
||||
Step 1:
|
||||
The slave volume will ask the master volume for a snapshot
|
||||
of (existing file entries, last offset, number of compacted times).
|
||||
|
||||
For each entry x in master existing file entries:
|
||||
if x does not exist locally:
|
||||
add x locally
|
||||
|
||||
For each entry y in local slave existing file entries:
|
||||
if y does not exist on master:
|
||||
delete y locally
|
||||
|
||||
Step 2:
|
||||
After this, use the last offset and number of compacted times to request
|
||||
the master volume to send a new file, and keep looping. If the number of
|
||||
compacted times is changed, go back to step 1 (very likely this can be
|
||||
optimized more later).
|
||||
|
||||
*/
|
||||
|
||||
func (v *Volume) Synchronize(volumeServer string, grpcDialOption grpc.DialOption) (err error) {
|
||||
var lastCompactRevision uint16 = 0
|
||||
var compactRevision uint16 = 0
|
||||
var masterMap *needle.CompactMap
|
||||
for i := 0; i < 3; i++ {
|
||||
if masterMap, _, compactRevision, err = fetchVolumeFileEntries(volumeServer, grpcDialOption, v.Id); err != nil {
|
||||
return fmt.Errorf("Failed to sync volume %d entries with %s: %v", v.Id, volumeServer, err)
|
||||
}
|
||||
if lastCompactRevision != compactRevision && lastCompactRevision != 0 {
|
||||
if err = v.Compact(0); err != nil {
|
||||
return fmt.Errorf("Compact Volume before synchronizing %v", err)
|
||||
}
|
||||
if err = v.commitCompact(); err != nil {
|
||||
return fmt.Errorf("Commit Compact before synchronizing %v", err)
|
||||
}
|
||||
}
|
||||
lastCompactRevision = compactRevision
|
||||
if err = v.trySynchronizing(volumeServer, grpcDialOption, masterMap, compactRevision); err == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type ByOffset []needle.NeedleValue
|
||||
|
||||
func (a ByOffset) Len() int { return len(a) }
|
||||
func (a ByOffset) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a ByOffset) Less(i, j int) bool { return a[i].Offset < a[j].Offset }
|
||||
|
||||
// trySynchronizing sync with remote volume server incrementally by
|
||||
// make up the local and remote delta.
|
||||
func (v *Volume) trySynchronizing(volumeServer string, grpcDialOption grpc.DialOption, masterMap *needle.CompactMap, compactRevision uint16) error {
|
||||
slaveIdxFile, err := os.Open(v.nm.IndexFileName())
|
||||
if err != nil {
|
||||
return fmt.Errorf("Open volume %d index file: %v", v.Id, err)
|
||||
}
|
||||
defer slaveIdxFile.Close()
|
||||
slaveMap, err := LoadBtreeNeedleMap(slaveIdxFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Load volume %d index file: %v", v.Id, err)
|
||||
}
|
||||
var delta []needle.NeedleValue
|
||||
if err := masterMap.Visit(func(needleValue needle.NeedleValue) error {
|
||||
if needleValue.Key == NeedleIdEmpty {
|
||||
return nil
|
||||
}
|
||||
if _, ok := slaveMap.Get(needleValue.Key); ok {
|
||||
return nil // skip intersection
|
||||
}
|
||||
delta = append(delta, needleValue)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("Add master entry: %v", err)
|
||||
}
|
||||
if err := slaveMap.m.Visit(func(needleValue needle.NeedleValue) error {
|
||||
if needleValue.Key == NeedleIdEmpty {
|
||||
return nil
|
||||
}
|
||||
if _, ok := masterMap.Get(needleValue.Key); ok {
|
||||
return nil // skip intersection
|
||||
}
|
||||
needleValue.Size = 0
|
||||
delta = append(delta, needleValue)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("Remove local entry: %v", err)
|
||||
}
|
||||
|
||||
// simulate to same ordering of remote .dat file needle entries
|
||||
sort.Sort(ByOffset(delta))
|
||||
|
||||
// make up the delta
|
||||
fetchCount := 0
|
||||
for _, needleValue := range delta {
|
||||
if needleValue.Size == 0 {
|
||||
// remove file entry from local
|
||||
v.removeNeedle(needleValue.Key)
|
||||
continue
|
||||
}
|
||||
// add master file entry to local data file
|
||||
if err := v.fetchNeedle(volumeServer, grpcDialOption, needleValue, compactRevision); err != nil {
|
||||
glog.V(0).Infof("Fetch needle %v from %s: %v", needleValue, volumeServer, err)
|
||||
return err
|
||||
}
|
||||
fetchCount++
|
||||
}
|
||||
glog.V(1).Infof("Fetched %d needles from %s", fetchCount, volumeServer)
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchVolumeFileEntries(volumeServer string, grpcDialOption grpc.DialOption, vid VolumeId) (m *needle.CompactMap, lastOffset uint64, compactRevision uint16, err error) {
|
||||
m = needle.NewCompactMap()
|
||||
|
||||
syncStatus, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
|
||||
if err != nil {
|
||||
return m, 0, 0, err
|
||||
}
|
||||
|
||||
total := 0
|
||||
err = operation.GetVolumeIdxEntries(volumeServer, grpcDialOption, uint32(vid), func(key NeedleId, offset Offset, size uint32) {
|
||||
// println("remote key", key, "offset", offset*NeedlePaddingSize, "size", size)
|
||||
if offset > 0 && size != TombstoneFileSize {
|
||||
m.Set(NeedleId(key), offset, size)
|
||||
} else {
|
||||
m.Delete(NeedleId(key))
|
||||
}
|
||||
total++
|
||||
})
|
||||
|
||||
glog.V(2).Infof("server %s volume %d, entries %d, last offset %d, revision %d", volumeServer, vid, total, syncStatus.TailOffset, syncStatus.CompactRevision)
|
||||
return m, syncStatus.TailOffset, uint16(syncStatus.CompactRevision), err
|
||||
|
||||
}
|
||||
|
||||
func (v *Volume) GetVolumeSyncStatus() *volume_server_pb.VolumeSyncStatusResponse {
|
||||
var syncStatus = &volume_server_pb.VolumeSyncStatusResponse{}
|
||||
if stat, err := v.dataFile.Stat(); err == nil {
|
||||
syncStatus.TailOffset = uint64(stat.Size())
|
||||
}
|
||||
syncStatus.Collection = v.Collection
|
||||
syncStatus.IdxFileSize = v.nm.IndexFileSize()
|
||||
syncStatus.CompactRevision = uint32(v.SuperBlock.CompactRevision)
|
||||
syncStatus.Ttl = v.SuperBlock.Ttl.String()
|
||||
syncStatus.Replication = v.SuperBlock.ReplicaPlacement.String()
|
||||
return syncStatus
|
||||
}
|
||||
|
||||
func (v *Volume) IndexFileContent() ([]byte, error) {
|
||||
return v.nm.IndexFileContent()
|
||||
}
|
||||
|
||||
// removeNeedle removes one needle by needle key
|
||||
func (v *Volume) removeNeedle(key NeedleId) {
|
||||
n := new(Needle)
|
||||
n.Id = key
|
||||
v.deleteNeedle(n)
|
||||
}
|
||||
|
||||
// fetchNeedle fetches a remote volume needle by vid, id, offset
|
||||
// The compact revision is checked first in case the remote volume
|
||||
// is compacted and the offset is invalid any more.
|
||||
func (v *Volume) fetchNeedle(volumeServer string, grpcDialOption grpc.DialOption, needleValue needle.NeedleValue, compactRevision uint16) error {
|
||||
|
||||
return operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
stream, err := client.VolumeSyncData(context.Background(), &volume_server_pb.VolumeSyncDataRequest{
|
||||
VolumeId: uint32(v.Id),
|
||||
Revision: uint32(compactRevision),
|
||||
Offset: uint32(needleValue.Offset),
|
||||
Size: uint32(needleValue.Size),
|
||||
NeedleId: needleValue.Key.String(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var fileContent []byte
|
||||
for {
|
||||
resp, err := stream.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read needle %v: %v", needleValue.Key.String(), err)
|
||||
}
|
||||
fileContent = append(fileContent, resp.FileContent...)
|
||||
}
|
||||
|
||||
offset, err := v.AppendBlob(fileContent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Appending volume %d error: %v", v.Id, err)
|
||||
}
|
||||
// println("add key", needleValue.Key, "offset", offset, "size", needleValue.Size)
|
||||
v.nm.Put(needleValue.Key, Offset(offset/NeedlePaddingSize), needleValue.Size)
|
||||
return nil
|
||||
})
|
||||
|
||||
}
|
||||
|
|
@ -38,7 +38,7 @@ func (v *Volume) Compact2() error {
|
|||
return v.copyDataBasedOnIndexFile(filePath+".cpd", filePath+".cpx")
|
||||
}
|
||||
|
||||
func (v *Volume) commitCompact() error {
|
||||
func (v *Volume) CommitCompact() error {
|
||||
glog.V(0).Infof("Committing volume %d vacuuming...", v.Id)
|
||||
v.dataFileAccessLock.Lock()
|
||||
defer v.dataFileAccessLock.Unlock()
|
||||
|
|
@ -53,7 +53,7 @@ func (v *Volume) commitCompact() error {
|
|||
|
||||
var e error
|
||||
if e = v.makeupDiff(v.FileName()+".cpd", v.FileName()+".cpx", v.FileName()+".dat", v.FileName()+".idx"); e != nil {
|
||||
glog.V(0).Infof("makeupDiff in commitCompact volume %d failed %v", v.Id, e)
|
||||
glog.V(0).Infof("makeupDiff in CommitCompact volume %d failed %v", v.Id, e)
|
||||
e = os.Remove(v.FileName() + ".cpd")
|
||||
if e != nil {
|
||||
return e
|
||||
|
|
@ -261,7 +261,7 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *Needle, offset int64) er
|
|||
}
|
||||
nv, ok := scanner.v.nm.Get(n.Id)
|
||||
glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
|
||||
if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {
|
||||
if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 && nv.Size != TombstoneFileSize {
|
||||
if err := scanner.nm.Put(n.Id, Offset(scanner.newOffset/NeedlePaddingSize), n.Size); err != nil {
|
||||
return fmt.Errorf("cannot put needle: %s", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ func TestCompaction(t *testing.T) {
|
|||
doSomeWritesDeletes(i+beforeCommitFileCount, v, t, infos)
|
||||
}
|
||||
|
||||
v.commitCompact()
|
||||
v.CommitCompact()
|
||||
|
||||
v.Close()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue