ADHOC: add read needle meta grpc (#3581)

* ADHOC: add read needle meta grpc

* add test

* nit

Co-authored-by: root <root@HQ-10MSTD3EY.roblox.local>
This commit is contained in:
Eric Yang 2022-09-06 23:51:27 -07:00 committed by GitHub
parent c07ab9c060
commit b324a6536c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1549 additions and 1123 deletions

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc-gen-go v1.28.1
// protoc v3.21.4
// source: filer.proto

View file

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.4
// source: filer.proto
package filer_pb
@ -11,6 +15,7 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedFilerClient is the client API for SeaweedFiler service.

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc-gen-go v1.28.1
// protoc v3.21.4
// source: iam.proto

View file

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.4
// source: iam.proto
package iam_pb
@ -8,6 +12,7 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service.

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc-gen-go v1.28.1
// protoc v3.21.4
// source: master.proto

View file

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.4
// source: master.proto
package master_pb
@ -11,6 +15,7 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedClient is the client API for Seaweed service.

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc-gen-go v1.28.1
// protoc v3.21.4
// source: mount.proto

View file

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.4
// source: mount.proto
package mount_pb
@ -11,6 +15,7 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedMountClient is the client API for SeaweedMount service.

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc-gen-go v1.28.1
// protoc v3.21.4
// source: mq.proto
@ -20,7 +20,7 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
//////////////////////////////////////////////////
// ////////////////////////////////////////////////
type SegmentInfo struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -617,7 +617,7 @@ func (x *CheckBrokerLoadResponse) GetBytesCount() int64 {
return 0
}
//////////////////////////////////////////////////
// ////////////////////////////////////////////////
type PublishRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache

View file

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.4
// source: mq.proto
package mq_pb
@ -11,6 +15,7 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedMessagingClient is the client API for SeaweedMessaging service.

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc-gen-go v1.28.1
// protoc v3.21.4
// source: remote.proto
@ -20,9 +20,9 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
/////////////////////////
// ///////////////////////
// Remote Storage related
/////////////////////////
// ///////////////////////
type RemoteConf struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache

View file

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.0
// protoc-gen-go v1.28.1
// protoc v3.21.4
// source: s3.proto

View file

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.4
// source: s3.proto
package s3_pb
@ -11,6 +15,7 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// SeaweedS3Client is the client API for SeaweedS3 service.

View file

@ -56,6 +56,8 @@ service VolumeServer {
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
}
rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
}
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
}
rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
@ -285,6 +287,19 @@ message ReadNeedleBlobResponse {
bytes needle_blob = 1;
}
message ReadNeedleMetaRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
int64 offset = 3; // actual offset
int32 size = 4;
}
message ReadNeedleMetaResponse {
uint32 cookie = 1;
uint64 last_modified = 2;
uint32 crc = 3;
string ttl = 4;
}
message WriteNeedleBlobRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.4
// source: volume_server.proto
package volume_server_pb
@ -11,13 +15,14 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// VolumeServerClient is the client API for VolumeServer service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type VolumeServerClient interface {
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
// Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
BatchDelete(ctx context.Context, in *BatchDeleteRequest, opts ...grpc.CallOption) (*BatchDeleteResponse, error)
VacuumVolumeCheck(ctx context.Context, in *VacuumVolumeCheckRequest, opts ...grpc.CallOption) (*VacuumVolumeCheckResponse, error)
VacuumVolumeCompact(ctx context.Context, in *VacuumVolumeCompactRequest, opts ...grpc.CallOption) (VolumeServer_VacuumVolumeCompactClient, error)
@ -39,6 +44,7 @@ type VolumeServerClient interface {
ReadVolumeFileStatus(ctx context.Context, in *ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*ReadVolumeFileStatusResponse, error)
CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error)
ReadNeedleBlob(ctx context.Context, in *ReadNeedleBlobRequest, opts ...grpc.CallOption) (*ReadNeedleBlobResponse, error)
ReadNeedleMeta(ctx context.Context, in *ReadNeedleMetaRequest, opts ...grpc.CallOption) (*ReadNeedleMetaResponse, error)
WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error)
ReadAllNeedles(ctx context.Context, in *ReadAllNeedlesRequest, opts ...grpc.CallOption) (VolumeServer_ReadAllNeedlesClient, error)
VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error)
@ -346,6 +352,15 @@ func (c *volumeServerClient) ReadNeedleBlob(ctx context.Context, in *ReadNeedleB
return out, nil
}
func (c *volumeServerClient) ReadNeedleMeta(ctx context.Context, in *ReadNeedleMetaRequest, opts ...grpc.CallOption) (*ReadNeedleMetaResponse, error) {
out := new(ReadNeedleMetaResponse)
err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/ReadNeedleMeta", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error) {
out := new(WriteNeedleBlobResponse)
err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/WriteNeedleBlob", in, out, opts...)
@ -677,7 +692,7 @@ func (c *volumeServerClient) Ping(ctx context.Context, in *PingRequest, opts ...
// All implementations must embed UnimplementedVolumeServerServer
// for forward compatibility
type VolumeServerServer interface {
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
// Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteResponse, error)
VacuumVolumeCheck(context.Context, *VacuumVolumeCheckRequest) (*VacuumVolumeCheckResponse, error)
VacuumVolumeCompact(*VacuumVolumeCompactRequest, VolumeServer_VacuumVolumeCompactServer) error
@ -699,6 +714,7 @@ type VolumeServerServer interface {
ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error)
CopyFile(*CopyFileRequest, VolumeServer_CopyFileServer) error
ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error)
ReadNeedleMeta(context.Context, *ReadNeedleMetaRequest) (*ReadNeedleMetaResponse, error)
WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error)
ReadAllNeedles(*ReadAllNeedlesRequest, VolumeServer_ReadAllNeedlesServer) error
VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error
@ -791,6 +807,9 @@ func (UnimplementedVolumeServerServer) CopyFile(*CopyFileRequest, VolumeServer_C
func (UnimplementedVolumeServerServer) ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleBlob not implemented")
}
func (UnimplementedVolumeServerServer) ReadNeedleMeta(context.Context, *ReadNeedleMetaRequest) (*ReadNeedleMetaResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleMeta not implemented")
}
func (UnimplementedVolumeServerServer) WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method WriteNeedleBlob not implemented")
}
@ -1239,6 +1258,24 @@ func _VolumeServer_ReadNeedleBlob_Handler(srv interface{}, ctx context.Context,
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_ReadNeedleMeta_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadNeedleMetaRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).ReadNeedleMeta(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/ReadNeedleMeta",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).ReadNeedleMeta(ctx, req.(*ReadNeedleMetaRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_WriteNeedleBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(WriteNeedleBlobRequest)
if err := dec(in); err != nil {
@ -1706,6 +1743,10 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
MethodName: "ReadNeedleBlob",
Handler: _VolumeServer_ReadNeedleBlob_Handler,
},
{
MethodName: "ReadNeedleMeta",
Handler: _VolumeServer_ReadNeedleMeta_Handler,
},
{
MethodName: "WriteNeedleBlob",
Handler: _VolumeServer_WriteNeedleBlob_Handler,

View file

@ -23,6 +23,35 @@ func (vs *VolumeServer) ReadNeedleBlob(ctx context.Context, req *volume_server_p
return resp, nil
}
func (vs *VolumeServer) ReadNeedleMeta(ctx context.Context, req *volume_server_pb.ReadNeedleMetaRequest) (resp *volume_server_pb.ReadNeedleMetaResponse, err error) {
resp = &volume_server_pb.ReadNeedleMetaResponse{}
volumeId := needle.VolumeId(req.VolumeId)
n := &needle.Needle{
Id: types.NeedleId(req.NeedleId),
Flags: 0x08,
}
size := req.Size
offset := req.Offset
hasVolume := vs.store.HasVolume(volumeId)
if !hasVolume {
return nil, fmt.Errorf("not found volume id %d and read needle metadata at ec shards is not supported", req.VolumeId)
}
err = vs.store.ReadVolumeNeedleMetaAt(volumeId, n, offset, size)
if err != nil {
return nil, err
}
resp.Cookie = uint32(n.Cookie)
resp.LastModified = n.LastModified
resp.Crc = n.Checksum.Value()
if n.HasTtl() {
resp.Ttl = n.Ttl.String()
}
return resp, nil
}
func (vs *VolumeServer) WriteNeedleBlob(ctx context.Context, req *volume_server_pb.WriteNeedleBlobRequest) (resp *volume_server_pb.WriteNeedleBlobResponse, err error) {
resp = &volume_server_pb.WriteNeedleBlobResponse{}
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))

View file

@ -210,10 +210,14 @@ func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle.
return types.Offset{}, 0, nil, fmt.Errorf("FindNeedleFromEcx: %v", err)
}
shard := ev.Shards[0]
intervals = ev.LocateEcShardNeedleInterval(version, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, version)))
return
}
func (ev *EcVolume) LocateEcShardNeedleInterval(version needle.Version, offset int64, size types.Size) (intervals []Interval) {
shard := ev.Shards[0]
// calculate the locations in the ec shards
intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shard.ecdFileSize, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, version)))
intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shard.ecdFileSize, offset, types.Size(needle.GetActualSize(size, version)))
return
}

View file

@ -209,8 +209,8 @@ func NeedleBodyLength(needleSize Size, version Version) int64 {
return int64(needleSize) + NeedleChecksumSize + int64(PaddingLength(needleSize, version))
}
//n should be a needle already read the header
//the input stream will read until next file entry
// n should be a needle already read the header
// the input stream will read until next file entry
func (n *Needle) ReadNeedleBody(r backend.BackendStorageFile, version Version, offset int64, bodyLength int64) (bytes []byte, err error) {
if bodyLength <= 0 {

View file

@ -74,7 +74,6 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
return ErrorSizeMismatch
}
}
n.DataSize = util.BytesToUint32(bytes[NeedleHeaderSize : NeedleHeaderSize+DataSizeSize])
startOffset := offset + NeedleHeaderSize + DataSizeSize + int64(n.DataSize)
@ -90,7 +89,6 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
if err != nil {
return err
}
var index int
index, err = n.readNeedleDataVersion2NonData(metaSlice)

View file

@ -384,6 +384,14 @@ func (s *Store) ReadVolumeNeedle(i needle.VolumeId, n *needle.Needle, readOption
}
return 0, fmt.Errorf("volume %d not found", i)
}
func (s *Store) ReadVolumeNeedleMetaAt(i needle.VolumeId, n *needle.Needle, offset int64, size int32) error {
if v := s.findVolume(i); v != nil {
return v.readNeedleMetaAt(n, offset, size)
}
return fmt.Errorf("volume %d not found", i)
}
func (s *Store) ReadVolumeNeedleDataInto(i needle.VolumeId, n *needle.Needle, readOption *ReadOption, writer io.Writer, offset int64, size int64) error {
if v := s.findVolume(i); v != nil {
return v.readNeedleDataInto(n, readOption, writer, offset, size)

View file

@ -80,6 +80,24 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize
return -1, ErrorNotFound
}
// read needle at a specific offset
func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) {
v.dataFileAccessLock.RLock()
defer v.dataFileAccessLock.RUnlock()
// read deleted meta data
if size < 0 {
size = -size
}
err = n.ReadNeedleMeta(v.DataBackend, offset, Size(size), v.Version())
if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
err = n.ReadNeedleMeta(v.DataBackend, offset+int64(MaxPossibleVolumeSize), Size(size), v.Version())
}
if err != nil {
return err
}
return nil
}
// read fills in Needle content by looking up n.Id from NeedleMapper
func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, writer io.Writer, offset int64, size int64) (err error) {
v.dataFileAccessLock.RLock()

View file

@ -0,0 +1,91 @@
package storage
import (
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/stretchr/testify/assert"
"testing"
)
func TestReadNeedMetaWithWritesAndUpdates(t *testing.T) {
dir := t.TempDir()
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0)
if err != nil {
t.Fatalf("volume creation: %v", err)
}
type WriteInfo struct {
offset int64
size int32
}
writeInfos := make([]WriteInfo, 30)
mockLastUpdateTime := uint64(1000000000000)
// initialize 20 needles then update first 10 needles
for i := 1; i <= 30; i++ {
n := newRandomNeedle(uint64(i % 20))
n.Flags = 0x08
n.LastModified = mockLastUpdateTime
mockLastUpdateTime += 2000
offset, _, _, err := v.writeNeedle2(n, true, false)
if err != nil {
t.Fatalf("write needle %d: %v", i, err)
}
writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(n.Size)}
}
expectedLastUpdateTime := uint64(1000000000000)
for i := 0; i < 30; i++ {
testNeedle := new(needle.Needle)
testNeedle.Id = types.Uint64ToNeedleId(uint64(i + 1%20))
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
expectedLastUpdateTime += 2000
}
}
func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
dir := t.TempDir()
v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0)
if err != nil {
t.Fatalf("volume creation: %v", err)
}
type WriteInfo struct {
offset int64
size int32
}
writeInfos := make([]WriteInfo, 10)
mockLastUpdateTime := uint64(1000000000000)
for i := 1; i <= 10; i++ {
n := newRandomNeedle(uint64(i % 5))
n.Flags = 0x08
n.LastModified = mockLastUpdateTime
mockLastUpdateTime += 2000
offset, _, _, err := v.writeNeedle2(n, true, false)
if err != nil {
t.Fatalf("write needle %d: %v", i, err)
}
if i < 5 {
size, err := v.deleteNeedle2(n)
if err != nil {
t.Fatalf("delete needle %d: %v", i, err)
}
writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(size)}
} else {
writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(n.Size)}
}
}
expectedLastUpdateTime := uint64(1000000000000)
for i := 0; i < 10; i++ {
testNeedle := new(needle.Needle)
testNeedle.Id = types.Uint64ToNeedleId(uint64(i + 1%5))
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
expectedLastUpdateTime += 2000
}
}