gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2590 from banjiaojuhao/padding_zero_for_web_read

Browse source
This commit is contained in:
Chris Lu 2022-01-13 09:46:23 -08:00 committed by GitHub
parent fe5b9e39cc 45e9c83421
commit 4cd4467179
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed/filer/reader_at.go
View file

@ -129,7 +129,7 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
}
if startOffset < chunk.LogicOffset {
gap := int(chunk.LogicOffset - startOffset)
glog.V(4).Infof("zero [%d,%d)", startOffset, startOffset+int64(gap))
glog.V(4).Infof("zero [%d,%d)", startOffset, chunk.LogicOffset)
n += int(min(int64(gap), remaining))
startOffset, remaining = chunk.LogicOffset, remaining-int64(gap)
if remaining <= 0 {

36
weed/filer/stream.go
View file

@ -80,11 +80,23 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ
fileId2Url[chunkView.FileId] = urlStrings
}
remaining := size
for _, chunkView := range chunkViews {
if offset < chunkView.LogicOffset {
gap := chunkView.LogicOffset - offset
remaining -= gap
glog.V(4).Infof("zero [%d,%d)", offset, chunkView.LogicOffset)
err := writeZero(writer, gap)
if err != nil {
return fmt.Errorf("write zero [%d,%d)", offset, chunkView.LogicOffset)
}
offset = chunkView.LogicOffset
}
urlStrings := fileId2Url[chunkView.FileId]
start := time.Now()
err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
offset += int64(chunkView.Size)
remaining -= int64(chunkView.Size)
stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
if err != nil {
stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
@ -92,6 +104,11 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ
}
stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
}
glog.V(4).Infof("zero [%d,%d)", offset, offset+remaining)
err := writeZero(writer, remaining)
if err != nil {
return fmt.Errorf("write zero [%d,%d)", offset, offset+remaining)
}
return nil
@ -99,6 +116,23 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ
// ---------------- ReadAllReader ----------------------------------
func writeZero(w io.Writer, size int64) (err error) {
zeroPadding := make([]byte, 1024)
var written int
for size > 0 {
if size > 1024 {
written, err = w.Write(zeroPadding)
} else {
written, err = w.Write(zeroPadding[:size])
}
size -= int64(written)
if err != nil {
return
}
}
return
}
func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {
buffer := bytes.Buffer{}