diff --git a/go.mod b/go.mod index 17cfe0bf6..ce985eb9d 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/klauspost/reedsolomon v1.11.8 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.9 - github.com/linxGnu/grocksdb v1.8.10 + github.com/linxGnu/grocksdb v1.8.11 github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ieproxy v0.0.11 // indirect @@ -107,7 +107,7 @@ require ( go.mongodb.org/mongo-driver v1.13.1 go.opencensus.io v0.24.0 // indirect gocloud.dev v0.36.0 - gocloud.dev/pubsub/natspubsub v0.35.0 + gocloud.dev/pubsub/natspubsub v0.36.0 gocloud.dev/pubsub/rabbitpubsub v0.36.0 golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d @@ -131,7 +131,7 @@ require ( modernc.org/mathutil v1.6.0 modernc.org/memory v1.7.2 // indirect modernc.org/opt v0.1.3 // indirect - modernc.org/sqlite v1.27.0 + modernc.org/sqlite v1.28.0 modernc.org/strutil v1.2.0 modernc.org/token v1.0.1 // indirect ) @@ -159,7 +159,7 @@ require ( github.com/shirou/gopsutil/v3 v3.23.11 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 - github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2 + github.com/ydb-platform/ydb-go-sdk/v3 v3.54.3 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 ) @@ -310,7 +310,7 @@ require ( github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect - github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a // indirect github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect diff --git a/go.sum b/go.sum index 8d2484b1b..9d2f3832c 100644 --- a/go.sum +++ b/go.sum @@ -636,8 +636,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/grocksdb v1.8.10 h1:6FAhBThErRfJaevGOZISYvkG7RD4gfzeq452X4r8pes= -github.com/linxGnu/grocksdb v1.8.10/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= +github.com/linxGnu/grocksdb v1.8.11 h1:BGol9e5gB1BrsTvOxloC88pe70TCqgrfLNwkyWW0kD8= +github.com/linxGnu/grocksdb v1.8.11/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -955,14 +955,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd h1:dzWP1Lu+A40W883dK/Mr3xyDSM/2MggS8GtHT0qgAnE= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a h1:9wx+kCrCQCdwmDe1AFW5yAHdzlo+RV7lcy6y7Zq661s= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 h1:IG5bPd+Lqyc+zsw2kmxqfGLkaDHuAEnWX63/8RBBiA4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0/go.mod h1:l6lZ+osdQOjDRBgRA4PQ06BuvmXN2neYjnRw8rCfd7s= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0= -github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2 h1:E0yUuuX7UmPxXm92+yQCjMveLFO3zfvYFIJVuAqsVRA= -github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2/go.mod h1:fjBLQ2TdQNl4bMjuWl9adoTGBypwUTPoGC+EqYqiIcU= +github.com/ydb-platform/ydb-go-sdk/v3 v3.54.3 h1:pxVqQ1bLW7PeotN7Ko/AvqUC3CDp7IpOzGNUyXU0YOE= +github.com/ydb-platform/ydb-go-sdk/v3 v3.54.3/go.mod h1:PMwXjf6joVP1ID0u0FBiDRh+DeLD1wx+HyM5zv/PzlE= github.com/ydb-platform/ydb-go-yc v0.10.2 h1:RAHy6g7ncxk1y0N4oS2MwYXLATqRqKBI6DYXuxpV2wo= github.com/ydb-platform/ydb-go-yc v0.10.2/go.mod h1:U1dX3LJy6zADId2DciCXlgrU/vphK1+CQzaefKq21dQ= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= @@ -1031,8 +1031,8 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= gocloud.dev v0.36.0 h1:q5zoXux4xkOZP473e1EZbG8Gq9f0vlg1VNH5Du/ybus= gocloud.dev v0.36.0/go.mod h1:bLxah6JQVKBaIxzsr5BQLYB4IYdWHkMZdzCXlo6F0gg= -gocloud.dev/pubsub/natspubsub v0.35.0 h1:qfZS9Gp9U9s2S5+JRbLkHJWlOFT568MAF1XwNLptB6g= -gocloud.dev/pubsub/natspubsub v0.35.0/go.mod h1:g/JZq8PDlkQtMJoAujEYduO49WsflxfMEaGp1GFIzuo= +gocloud.dev/pubsub/natspubsub v0.36.0 h1:QGBT53UN7VO7zttlGhAcLrwXjv0BKUbMET7hIlq5ehs= +gocloud.dev/pubsub/natspubsub v0.36.0/go.mod h1:OqWUmvFVNlPFF4YQ2PS7+/TnsEv/2loNfC1Uyp2plfA= gocloud.dev/pubsub/rabbitpubsub v0.36.0 h1:f7KXLhk5HVZ81a2LKfTQY/HBDyXF0teQ7aMuJN+iK7I= gocloud.dev/pubsub/rabbitpubsub v0.36.0/go.mod h1:xCbgXeLHXrb2Yqs3WOTvmTvk6WeIfMMn3uq34uOk9yM= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= @@ -1549,8 +1549,8 @@ modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.27.0 h1:MpKAHoyYB7xqcwnUwkuD+npwEa0fojF0B5QRbN+auJ8= -modernc.org/sqlite v1.27.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= +modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= +modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= diff --git a/weed/storage/needle/needle_read.go b/weed/storage/needle/needle_read.go index 19b801bd4..1041e924d 100644 --- a/weed/storage/needle/needle_read.go +++ b/weed/storage/needle/needle_read.go @@ -93,7 +93,17 @@ func (n *Needle) ReadData(r backend.BackendStorageFile, offset int64, size Size, if err != nil { return err } - return n.ReadBytes(bytes, offset, size, version) + + err = n.ReadBytes(bytes, offset, size, version) + if err == ErrorSizeMismatch && OffsetSize == 4 { + offset = offset + int64(MaxPossibleVolumeSize) + bytes, err = ReadNeedleBlob(r, offset, size, version) + if err != nil { + return err + } + err = n.ReadBytes(bytes, offset, size, version) + } + return err } func (n *Needle) ParseNeedleHeader(bytes []byte) { diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index e10fedf9b..0f92a2e93 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -60,6 +60,9 @@ func doCheckAndFixVolumeData(v *Volume, indexFile *os.File, indexOffset int64) ( } } else { if lastAppendAtNs, err = verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToActualOffset(), key, size); err != nil { + if err == ErrorSizeMismatch { + return verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToActualOffset()+int64(MaxPossibleVolumeSize), key, size) + } return lastAppendAtNs, err } } diff --git a/weed/storage/volume_read.go b/weed/storage/volume_read.go index 8e9717689..09fbdd701 100644 --- a/weed/storage/volume_read.go +++ b/weed/storage/volume_read.go @@ -55,9 +55,6 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize } if readOption == nil || !readOption.IsMetaOnly { err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version()) - if err == needle.ErrorSizeMismatch && OffsetSize == 4 { - err = n.ReadData(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) - } v.checkReadWriteError(err) if err != nil { return 0, err