gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Include meta in ReadAllNeedles (#3991)

Browse source 
This is useful for doing backups on the data so we can accurately store the
last modified time, the compression state, and verify the crc.

Previously we were doing VolumeNeedleStatus and then an HTTP request which
needlessly read from the dat file twice.
This commit is contained in:
James Hartig 2022-11-20 23:19:41 -05:00 committed by GitHub
parent 70a4c98b00
commit 4c85da7844
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 746 additions and 708 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
unmaintained/stream_read_volume/stream_read_volume.go
View file

@ -5,13 +5,14 @@ import (
"errors"
"flag"
"fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"io"
)
var (
@ -29,7 +30,7 @@ func main() {
vid := uint32(*volumeId)
eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
fmt.Printf("%d,%x%08x %d %v %d %x\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob), resp.NeedleBlobCompressed, resp.LastModified, resp.Crc)
return nil
}

3
weed/pb/volume_server.proto
View file

@ -317,6 +317,9 @@ message ReadAllNeedlesResponse {
uint64 needle_id = 2;
uint32 cookie = 3;
bytes needle_blob = 5;
bool needle_blob_compressed = 6;
uint64 last_modified = 7;
uint32 crc = 8;
}
message VolumeTailSenderRequest {

1435
weed/pb/volume_server_pb/volume_server.pb.go
View file

File diff suppressed because it is too large Load diff

11
weed/storage/volume_read_all.go
View file

@ -30,10 +30,13 @@ func (scanner *VolumeFileScanner4ReadAll) VisitNeedle(n *needle.Needle, offset i
}
sendErr := scanner.Stream.Send(&volume_server_pb.ReadAllNeedlesResponse{
VolumeId: uint32(scanner.V.Id),
NeedleId: uint64(n.Id),
Cookie: uint32(n.Cookie),
NeedleBlob: n.Data,
VolumeId: uint32(scanner.V.Id),
NeedleId: uint64(n.Id),
Cookie: uint32(n.Cookie),
NeedleBlob: n.Data,
NeedleBlobCompressed: n.IsCompressed(),
LastModified: n.LastModified,
Crc: n.Checksum.Value(),
})
if sendErr != nil {
return sendErr