seaweedfs/weed/filer/stream.go

393 lines
11 KiB
Go
Raw Normal View History

2020-09-01 07:21:19 +00:00
package filer
import (
"bytes"
2020-10-13 20:53:34 +00:00
"fmt"
"golang.org/x/exp/slices"
"io"
"math"
2021-08-07 22:35:27 +00:00
"sort"
"strings"
2021-08-26 00:28:50 +00:00
"sync"
2021-05-24 07:14:50 +00:00
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
var getLookupFileIdBackoffSchedule = []time.Duration{
150 * time.Millisecond,
600 * time.Millisecond,
1800 * time.Millisecond,
}
2021-08-16 02:46:45 +00:00
func HasData(entry *filer_pb.Entry) bool {
if len(entry.Content) > 0 {
return true
}
return len(entry.Chunks) > 0
}
func IsSameData(a, b *filer_pb.Entry) bool {
if len(a.Content) > 0 || len(b.Content) > 0 {
return bytes.Equal(a.Content, b.Content)
}
return isSameChunks(a.Chunks, b.Chunks)
}
func isSameChunks(a, b []*filer_pb.FileChunk) bool {
if len(a) != len(b) {
return false
}
slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool {
return strings.Compare(i.ETag, j.ETag) < 0
})
slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool {
return strings.Compare(i.ETag, j.ETag) < 0
})
2021-08-16 02:46:45 +00:00
for i := 0; i < len(a); i++ {
if a[i].ETag != b[i].ETag {
2021-08-16 02:46:45 +00:00
return false
}
}
return true
}
func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.Reader {
if len(entry.Content) > 0 {
return bytes.NewReader(entry.Content)
}
return NewChunkStreamReader(filerClient, entry.Chunks)
}
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
return StreamContentWithThrottler(masterClient, writer, chunks, offset, size, 0)
}
func StreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) error {
2022-03-14 10:19:16 +00:00
glog.V(4).Infof("start to stream content for chunks: %+v", chunks)
chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
fileId2Url := make(map[string][]string)
for _, chunkView := range chunkViews {
var urlStrings []string
var err error
for _, backoff := range getLookupFileIdBackoffSchedule {
urlStrings, err = masterClient.GetLookupFileIdFunction()(chunkView.FileId)
2022-07-26 06:56:45 +00:00
if err == nil && len(urlStrings) > 0 {
break
}
2022-07-26 06:56:45 +00:00
time.Sleep(backoff)
}
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
return err
2021-03-11 18:34:36 +00:00
} else if len(urlStrings) == 0 {
2022-07-15 10:29:15 +00:00
errUrlNotFound := fmt.Errorf("operation LookupFileId %s failed, err: urls not found", chunkView.FileId)
glog.Error(errUrlNotFound)
return errUrlNotFound
}
fileId2Url[chunkView.FileId] = urlStrings
}
downloadThrottler := util.NewWriteThrottler(downloadMaxBytesPs)
2022-01-13 14:20:33 +00:00
remaining := size
for _, chunkView := range chunkViews {
2022-01-13 14:20:33 +00:00
if offset < chunkView.LogicOffset {
gap := chunkView.LogicOffset - offset
remaining -= gap
glog.V(4).Infof("zero [%d,%d)", offset, chunkView.LogicOffset)
err := writeZero(writer, gap)
if err != nil {
return fmt.Errorf("write zero [%d,%d)", offset, chunkView.LogicOffset)
}
offset = chunkView.LogicOffset
}
urlStrings := fileId2Url[chunkView.FileId]
2021-05-24 07:14:50 +00:00
start := time.Now()
err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
2022-01-13 14:20:33 +00:00
offset += int64(chunkView.Size)
remaining -= int64(chunkView.Size)
2021-05-24 07:14:50 +00:00
stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
2020-10-10 23:02:10 +00:00
if err != nil {
2021-05-24 07:14:50 +00:00
stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
2020-10-13 20:53:34 +00:00
return fmt.Errorf("read chunk: %v", err)
}
2021-05-24 07:14:50 +00:00
stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
downloadThrottler.MaybeSlowdown(int64(chunkView.Size))
}
2022-03-14 10:19:16 +00:00
if remaining > 0 {
glog.V(4).Infof("zero [%d,%d)", offset, offset+remaining)
err := writeZero(writer, remaining)
if err != nil {
return fmt.Errorf("write zero [%d,%d)", offset, offset+remaining)
}
2022-01-13 14:20:33 +00:00
}
return nil
}
2020-04-28 07:05:47 +00:00
// ---------------- ReadAllReader ----------------------------------
2022-01-13 14:20:33 +00:00
func writeZero(w io.Writer, size int64) (err error) {
zeroPadding := make([]byte, 1024)
var written int
for size > 0 {
if size > 1024 {
written, err = w.Write(zeroPadding)
} else {
written, err = w.Write(zeroPadding[:size])
}
size -= int64(written)
if err != nil {
return
}
}
return
}
2022-02-26 10:16:47 +00:00
func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) error {
2020-04-28 07:05:47 +00:00
lookupFileIdFn := func(fileId string) (targetUrls []string, err error) {
2020-04-28 07:05:47 +00:00
return masterClient.LookupFileId(fileId)
}
2022-02-26 10:16:47 +00:00
chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, int64(len(buffer)))
idx := 0
2020-04-28 07:05:47 +00:00
for _, chunkView := range chunkViews {
urlStrings, err := lookupFileIdFn(chunkView.FileId)
2020-04-28 07:05:47 +00:00
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
2022-02-26 10:16:47 +00:00
return err
2020-04-28 07:05:47 +00:00
}
2020-10-09 06:19:42 +00:00
2022-02-26 10:16:47 +00:00
n, err := retriedFetchChunkData(buffer[idx:idx+int(chunkView.Size)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset)
2020-10-09 06:19:42 +00:00
if err != nil {
2022-02-26 10:16:47 +00:00
return err
2020-04-28 07:05:47 +00:00
}
2022-02-26 10:16:47 +00:00
idx += n
2020-04-28 07:05:47 +00:00
}
2022-02-26 10:16:47 +00:00
return nil
2020-04-28 07:05:47 +00:00
}
// ---------------- ChunkStreamReader ----------------------------------
type ChunkStreamReader struct {
2021-09-01 09:45:42 +00:00
chunkViews []*ChunkView
totalSize int64
logicOffset int64
buffer []byte
bufferOffset int64
bufferLock sync.Mutex
chunk string
lookupFileId wdclient.LookupFileIdFunctionType
}
var _ = io.ReadSeeker(&ChunkStreamReader{})
var _ = io.ReaderAt(&ChunkStreamReader{})
2021-08-07 22:35:27 +00:00
func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
slices.SortFunc(chunkViews, func(a, b *ChunkView) bool {
return a.LogicOffset < b.LogicOffset
2021-08-07 22:35:27 +00:00
})
2021-08-07 21:46:23 +00:00
var totalSize int64
for _, chunk := range chunkViews {
totalSize += int64(chunk.Size)
}
return &ChunkStreamReader{
chunkViews: chunkViews,
lookupFileId: lookupFileIdFn,
2021-08-07 21:46:23 +00:00
totalSize: totalSize,
}
}
2021-08-07 22:35:27 +00:00
func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
2021-08-07 22:35:27 +00:00
lookupFileIdFn := func(fileId string) (targetUrl []string, err error) {
return masterClient.LookupFileId(fileId)
}
2021-08-07 22:35:27 +00:00
return doNewChunkStreamReader(lookupFileIdFn, chunks)
}
2021-08-07 22:35:27 +00:00
func NewChunkStreamReader(filerClient filer_pb.FilerClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
2021-08-07 21:46:23 +00:00
2021-08-07 22:35:27 +00:00
lookupFileIdFn := LookupFn(filerClient)
return doNewChunkStreamReader(lookupFileIdFn, chunks)
}
func (c *ChunkStreamReader) ReadAt(p []byte, off int64) (n int, err error) {
2021-08-26 00:28:50 +00:00
c.bufferLock.Lock()
defer c.bufferLock.Unlock()
if err = c.prepareBufferFor(off); err != nil {
return
}
c.logicOffset = off
2021-08-26 00:28:50 +00:00
return c.doRead(p)
}
func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
2021-08-26 00:28:50 +00:00
c.bufferLock.Lock()
defer c.bufferLock.Unlock()
return c.doRead(p)
}
func (c *ChunkStreamReader) doRead(p []byte) (n int, err error) {
// fmt.Printf("do read [%d,%d) at %s[%d,%d)\n", c.logicOffset, c.logicOffset+int64(len(p)), c.chunk, c.bufferOffset, c.bufferOffset+int64(len(c.buffer)))
2020-04-29 09:42:58 +00:00
for n < len(p) {
2021-08-26 00:28:50 +00:00
// println("read", c.logicOffset)
if err = c.prepareBufferFor(c.logicOffset); err != nil {
return
}
2021-08-26 00:28:50 +00:00
t := copy(p[n:], c.buffer[c.logicOffset-c.bufferOffset:])
2020-04-29 09:42:58 +00:00
n += t
2021-08-08 08:21:42 +00:00
c.logicOffset += int64(t)
}
return
}
func (c *ChunkStreamReader) isBufferEmpty() bool {
2021-09-01 09:45:42 +00:00
return len(c.buffer) <= int(c.logicOffset-c.bufferOffset)
}
func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
2021-08-26 00:28:50 +00:00
c.bufferLock.Lock()
defer c.bufferLock.Unlock()
var err error
switch whence {
case io.SeekStart:
case io.SeekCurrent:
2021-08-08 08:21:42 +00:00
offset += c.logicOffset
case io.SeekEnd:
2021-08-07 21:46:23 +00:00
offset = c.totalSize + offset
}
2021-08-07 21:46:23 +00:00
if offset > c.totalSize {
err = io.ErrUnexpectedEOF
2021-08-08 08:21:42 +00:00
} else {
c.logicOffset = offset
}
2021-08-08 08:21:42 +00:00
return offset, err
}
2021-08-26 00:28:50 +00:00
func insideChunk(offset int64, chunk *ChunkView) bool {
return chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size)
}
2021-08-08 08:21:42 +00:00
func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
2021-08-07 22:35:27 +00:00
// stay in the same chunk
2021-08-26 00:28:50 +00:00
if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
return nil
2021-08-07 22:35:27 +00:00
}
2021-08-26 00:28:50 +00:00
// fmt.Printf("fetch for offset %d\n", offset)
2021-08-07 22:35:27 +00:00
// need to seek to a different chunk
currentChunkIndex := sort.Search(len(c.chunkViews), func(i int) bool {
2021-08-16 06:07:58 +00:00
return offset < c.chunkViews[i].LogicOffset
2021-08-07 22:35:27 +00:00
})
if currentChunkIndex == len(c.chunkViews) {
2021-08-16 06:07:58 +00:00
// not found
2021-08-26 00:28:50 +00:00
if insideChunk(offset, c.chunkViews[0]) {
// fmt.Printf("select0 chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
2021-08-16 06:07:58 +00:00
currentChunkIndex = 0
2021-08-26 00:28:50 +00:00
} else if insideChunk(offset, c.chunkViews[len(c.chunkViews)-1]) {
currentChunkIndex = len(c.chunkViews) - 1
// fmt.Printf("select last chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
2021-08-16 06:07:58 +00:00
} else {
return io.EOF
}
} else if currentChunkIndex > 0 {
2021-08-26 00:28:50 +00:00
if insideChunk(offset, c.chunkViews[currentChunkIndex]) {
// good hit
2021-09-01 09:45:42 +00:00
} else if insideChunk(offset, c.chunkViews[currentChunkIndex-1]) {
2021-08-16 06:07:58 +00:00
currentChunkIndex -= 1
2021-08-26 00:28:50 +00:00
// fmt.Printf("select -1 chunk %d %s\n", currentChunkIndex, c.chunkViews[currentChunkIndex].FileId)
2021-08-16 06:07:58 +00:00
} else {
2021-08-26 00:28:50 +00:00
// glog.Fatalf("unexpected1 offset %d", offset)
2021-08-16 06:07:58 +00:00
return fmt.Errorf("unexpected1 offset %d", offset)
}
} else {
2021-08-26 00:28:50 +00:00
// glog.Fatalf("unexpected2 offset %d", offset)
2021-08-16 06:07:58 +00:00
return fmt.Errorf("unexpected2 offset %d", offset)
2021-08-07 22:35:27 +00:00
}
// positioning within the new chunk
chunk := c.chunkViews[currentChunkIndex]
2021-08-26 00:28:50 +00:00
if insideChunk(offset, chunk) {
2021-08-07 22:35:27 +00:00
if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
2021-08-08 08:21:42 +00:00
if err = c.fetchChunkToBuffer(chunk); err != nil {
return
}
}
2021-08-26 00:28:50 +00:00
} else {
// glog.Fatalf("unexpected3 offset %d in %s [%d,%d)", offset, chunk.FileId, chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
return fmt.Errorf("unexpected3 offset %d in %s [%d,%d)", offset, chunk.FileId, chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
}
2021-08-08 08:21:42 +00:00
return
}
func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
urlStrings, err := c.lookupFileId(chunkView.FileId)
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
return err
}
var buffer bytes.Buffer
2020-10-13 07:29:46 +00:00
var shouldRetry bool
for _, urlString := range urlStrings {
shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
buffer.Write(data)
})
2020-10-13 07:29:46 +00:00
if !shouldRetry {
break
}
if err != nil {
glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
buffer.Reset()
} else {
break
}
}
if err != nil {
return err
}
c.buffer = buffer.Bytes()
c.bufferOffset = chunkView.LogicOffset
2021-08-26 00:28:50 +00:00
c.chunk = chunkView.FileId
2021-08-26 00:28:50 +00:00
// glog.V(0).Infof("fetched %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
return nil
}
2020-04-28 07:05:47 +00:00
func (c *ChunkStreamReader) Close() {
// TODO try to release and reuse buffer
}
2020-03-23 07:01:34 +00:00
func VolumeId(fileId string) string {
lastCommaIndex := strings.LastIndex(fileId, ",")
if lastCommaIndex > 0 {
return fileId[:lastCommaIndex]
}
2020-03-23 07:01:34 +00:00
return fileId
}