seaweedfs/weed/filer/stream.go

228 lines
6.1 KiB
Go
Raw Normal View History

2020-09-01 07:21:19 +00:00
package filer
import (
"bytes"
2020-10-13 20:53:34 +00:00
"fmt"
"io"
"math"
"strings"
2021-05-24 07:14:50 +00:00
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
2021-05-24 07:14:50 +00:00
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient"
)
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
2021-03-11 17:38:59 +00:00
glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
fileId2Url := make(map[string][]string)
for _, chunkView := range chunkViews {
urlStrings, err := masterClient.GetLookupFileIdFunction()(chunkView.FileId)
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
return err
2021-03-11 18:34:36 +00:00
} else if len(urlStrings) == 0 {
glog.Errorf("operation LookupFileId %s failed, err: urls not found", chunkView.FileId)
return fmt.Errorf("operation LookupFileId %s failed, err: urls not found", chunkView.FileId)
}
fileId2Url[chunkView.FileId] = urlStrings
}
for _, chunkView := range chunkViews {
urlStrings := fileId2Url[chunkView.FileId]
2021-05-24 07:14:50 +00:00
start := time.Now()
2021-03-23 05:13:19 +00:00
data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
2021-05-24 07:14:50 +00:00
stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
2020-10-10 23:02:10 +00:00
if err != nil {
2021-05-24 07:14:50 +00:00
stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
2020-10-13 20:53:34 +00:00
return fmt.Errorf("read chunk: %v", err)
}
2020-10-13 20:53:34 +00:00
_, err = w.Write(data)
if err != nil {
2021-05-24 07:14:50 +00:00
stats.FilerRequestCounter.WithLabelValues("chunkDownloadedError").Inc()
2020-10-13 20:53:34 +00:00
return fmt.Errorf("write chunk: %v", err)
}
2021-05-24 07:14:50 +00:00
stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
}
return nil
}
2020-04-28 07:05:47 +00:00
// ---------------- ReadAllReader ----------------------------------
func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {
buffer := bytes.Buffer{}
lookupFileIdFn := func(fileId string) (targetUrls []string, err error) {
2020-04-28 07:05:47 +00:00
return masterClient.LookupFileId(fileId)
}
chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
2020-04-28 07:05:47 +00:00
for _, chunkView := range chunkViews {
urlStrings, err := lookupFileIdFn(chunkView.FileId)
2020-04-28 07:05:47 +00:00
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
return nil, err
}
2020-10-09 06:19:42 +00:00
2021-03-23 05:13:19 +00:00
data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
2020-10-09 06:19:42 +00:00
if err != nil {
return nil, err
2020-04-28 07:05:47 +00:00
}
2020-10-09 06:19:42 +00:00
buffer.Write(data)
2020-04-28 07:05:47 +00:00
}
return buffer.Bytes(), nil
}
// ---------------- ChunkStreamReader ----------------------------------
type ChunkStreamReader struct {
chunkViews []*ChunkView
logicOffset int64
buffer []byte
bufferOffset int64
bufferPos int
chunkIndex int
lookupFileId wdclient.LookupFileIdFunctionType
}
var _ = io.ReadSeeker(&ChunkStreamReader{})
func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
lookupFileIdFn := func(fileId string) (targetUrl []string, err error) {
return masterClient.LookupFileId(fileId)
}
chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
return &ChunkStreamReader{
chunkViews: chunkViews,
lookupFileId: lookupFileIdFn,
}
}
func NewChunkStreamReader(filerClient filer_pb.FilerClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
lookupFileIdFn := LookupFn(filerClient)
chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
return &ChunkStreamReader{
2020-05-10 10:50:30 +00:00
chunkViews: chunkViews,
lookupFileId: lookupFileIdFn,
}
}
func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
2020-04-29 09:42:58 +00:00
for n < len(p) {
if c.isBufferEmpty() {
if c.chunkIndex >= len(c.chunkViews) {
return n, io.EOF
}
chunkView := c.chunkViews[c.chunkIndex]
c.fetchChunkToBuffer(chunkView)
c.chunkIndex++
}
2020-04-29 09:42:58 +00:00
t := copy(p[n:], c.buffer[c.bufferPos:])
c.bufferPos += t
n += t
}
return
}
func (c *ChunkStreamReader) isBufferEmpty() bool {
return len(c.buffer) <= c.bufferPos
}
func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
2020-03-22 06:48:11 +00:00
var totalSize int64
for _, chunk := range c.chunkViews {
totalSize += int64(chunk.Size)
}
var err error
switch whence {
case io.SeekStart:
case io.SeekCurrent:
offset += c.bufferOffset + int64(c.bufferPos)
case io.SeekEnd:
2020-03-22 06:48:11 +00:00
offset = totalSize + offset
}
2020-03-22 06:48:11 +00:00
if offset > totalSize {
err = io.ErrUnexpectedEOF
}
for i, chunk := range c.chunkViews {
if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
c.fetchChunkToBuffer(chunk)
c.chunkIndex = i + 1
break
}
}
}
c.bufferPos = int(offset - c.bufferOffset)
return offset, err
}
func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
urlStrings, err := c.lookupFileId(chunkView.FileId)
if err != nil {
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
return err
}
var buffer bytes.Buffer
2020-10-13 07:29:46 +00:00
var shouldRetry bool
for _, urlString := range urlStrings {
shouldRetry, err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
buffer.Write(data)
})
2020-10-13 07:29:46 +00:00
if !shouldRetry {
break
}
if err != nil {
glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
buffer.Reset()
} else {
break
}
}
if err != nil {
return err
}
c.buffer = buffer.Bytes()
c.bufferPos = 0
c.bufferOffset = chunkView.LogicOffset
// glog.V(0).Infof("read %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
return nil
}
2020-04-28 07:05:47 +00:00
func (c *ChunkStreamReader) Close() {
// TODO try to release and reuse buffer
}
2020-03-23 07:01:34 +00:00
func VolumeId(fileId string) string {
lastCommaIndex := strings.LastIndex(fileId, ",")
if lastCommaIndex > 0 {
return fileId[:lastCommaIndex]
}
2020-03-23 07:01:34 +00:00
return fileId
}