gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

ADHOC: filter deleted files from idx file binary search (#3763)

Browse source 
* ADHOC: filter deleted files from idx file binary search

* remove unwanted check

Co-authored-by: root <root@HQ-10MSTD3EY.roblox.local>
This commit is contained in:
Eric Yang 2022-09-29 12:48:36 -07:00 committed by GitHub
parent 3de1e19780
commit 56c94cc08e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
weed/shell/command_volume_fsck.go
View file

@ -393,8 +393,9 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI
} }
buf.Write(resp.FileContent) buf.Write(resp.FileContent)
} }
fileredBuf := filterDeletedNeedleFromIdx(buf.Bytes())
if vinfo.isReadOnly == false { if vinfo.isReadOnly == false {
index, err := idx.FirstInvalidIndex(buf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) { index, err := idx.FirstInvalidIndex(fileredBuf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{ resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{
VolumeId: volumeId, VolumeId: volumeId,
NeedleId: uint64(key), NeedleId: uint64(key),
@ -409,11 +410,11 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI
if err != nil { if err != nil {
fmt.Fprintf(writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err) fmt.Fprintf(writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err)
} else { } else {
buf.Truncate(index * types.NeedleMapEntrySize) fileredBuf.Truncate(index * types.NeedleMapEntrySize)
} }
} }
idxFilename := getVolumeFileIdFile(tempFolder, dataNodeId, volumeId) idxFilename := getVolumeFileIdFile(tempFolder, dataNodeId, volumeId)
err = writeToFile(buf.Bytes(), idxFilename) err = writeToFile(fileredBuf.Bytes(), idxFilename)
if err != nil { if err != nil {
return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err) return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err)
} }
@ -719,3 +720,14 @@ func writeToFile(bytes []byte, fileName string) error {
dst.Write(bytes) dst.Write(bytes)
return nil return nil
} }
func filterDeletedNeedleFromIdx(arr []byte) bytes.Buffer {
var filteredBuf bytes.Buffer
for i := 0; i < len(arr); i += types.NeedleMapEntrySize {
size := types.BytesToSize(arr[i+types.NeedleIdSize+types.OffsetSize : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize])
if size > 0 {
filteredBuf.Write(arr[i : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize])
}
}
return filteredBuf
}

4
weed/storage/volume_read.go
View file

@ -84,10 +84,6 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize
func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) { func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) {
v.dataFileAccessLock.RLock() v.dataFileAccessLock.RLock()
defer v.dataFileAccessLock.RUnlock() defer v.dataFileAccessLock.RUnlock()
// read deleted meta data
if size < 0 {
size = -size
}
err = n.ReadNeedleMeta(v.DataBackend, offset, Size(size), v.Version()) err = n.ReadNeedleMeta(v.DataBackend, offset, Size(size), v.Version())
if err == needle.ErrorSizeMismatch && OffsetSize == 4 { if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
err = n.ReadNeedleMeta(v.DataBackend, offset+int64(MaxPossibleVolumeSize), Size(size), v.Version()) err = n.ReadNeedleMeta(v.DataBackend, offset+int64(MaxPossibleVolumeSize), Size(size), v.Version())