gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

fix aws style Etag for chunks

Browse source
This commit is contained in:
Konstantin Lebedev 2021-04-28 22:28:05 +05:00
parent ccbe02218a
commit c2269123d3
3 changed files with 9 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
weed/filer/filechunks.go
View file

@ -2,7 +2,6 @@ package filer
import ( import (
"bytes" "bytes"
"encoding/hex"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/chrislusf/seaweedfs/weed/wdclient"
"math" "math"
@ -43,12 +42,11 @@ func ETagEntry(entry *Entry) (etag string) {
func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) { func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
if len(chunks) == 1 { if len(chunks) == 1 {
return chunks[0].ETag return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag))
} }
md5_digests := [][]byte{} md5_digests := [][]byte{}
for _, c := range chunks { for _, c := range chunks {
md5_decoded, _ := hex.DecodeString(c.ETag) md5_digests = append(md5_digests, util.Base64Md5ToBytes(c.ETag))
md5_digests = append(md5_digests, md5_decoded)
} }
return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks)) return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks))
} }

2
weed/operation/upload_content.go
View file

@ -39,7 +39,7 @@ func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *fi
Offset: offset, Offset: offset,
Size: uint64(uploadResult.Size), Size: uint64(uploadResult.Size),
Mtime: time.Now().UnixNano(), Mtime: time.Now().UnixNano(),
ETag: uploadResult.ETag, ETag: uploadResult.ContentMd5,
CipherKey: uploadResult.CipherKey, CipherKey: uploadResult.CipherKey,
IsCompressed: uploadResult.Gzip > 0, IsCompressed: uploadResult.Gzip > 0,
Fid: fid, Fid: fid,

6
weed/server/filer_server_handlers_write_upload.go
View file

@ -1,6 +1,7 @@
package weed_server package weed_server
import ( import (
"bytes"
"crypto/md5" "crypto/md5"
"hash" "hash"
"io" "io"
@ -71,6 +72,11 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
if uploadResult.Size == 0 { if uploadResult.Size == 0 {
break break
} }
uploadedMd5 := util.Base64Md5ToBytes(uploadResult.ContentMd5)
readedMd5 := md5Hash.Sum(nil)
if !bytes.Equal(uploadedMd5, readedMd5) {
glog.Errorf("md5 %x does not match %x uploaded chunk %s to the volume server", readedMd5, uploadedMd5, uploadResult.Name)
}
// Save to chunk manifest structure // Save to chunk manifest structure
fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset)) fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset))