2019-06-05 08:30:24 +00:00
|
|
|
package filer2
|
|
|
|
|
|
|
|
import (
|
2020-03-21 03:31:11 +00:00
|
|
|
"bytes"
|
2019-06-05 08:30:24 +00:00
|
|
|
"io"
|
2020-03-21 03:31:11 +00:00
|
|
|
"math"
|
2020-03-22 08:00:36 +00:00
|
|
|
"strings"
|
2019-06-05 08:30:24 +00:00
|
|
|
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
|
|
|
)
|
|
|
|
|
2020-03-22 08:37:46 +00:00
|
|
|
func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
|
2019-06-05 08:30:24 +00:00
|
|
|
|
|
|
|
chunkViews := ViewFromChunks(chunks, offset, size)
|
|
|
|
|
|
|
|
fileId2Url := make(map[string]string)
|
|
|
|
|
|
|
|
for _, chunkView := range chunkViews {
|
|
|
|
|
|
|
|
urlString, err := masterClient.LookupFileId(chunkView.FileId)
|
|
|
|
if err != nil {
|
|
|
|
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
fileId2Url[chunkView.FileId] = urlString
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, chunkView := range chunkViews {
|
2020-03-09 04:39:33 +00:00
|
|
|
|
2019-06-05 08:30:24 +00:00
|
|
|
urlString := fileId2Url[chunkView.FileId]
|
2020-04-14 04:58:10 +00:00
|
|
|
err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
|
2019-06-05 08:30:24 +00:00
|
|
|
w.Write(data)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
2020-03-21 03:31:11 +00:00
|
|
|
|
|
|
|
type ChunkStreamReader struct {
|
|
|
|
masterClient *wdclient.MasterClient
|
|
|
|
chunkViews []*ChunkView
|
|
|
|
logicOffset int64
|
2020-03-22 05:16:00 +00:00
|
|
|
buffer []byte
|
2020-03-21 03:31:11 +00:00
|
|
|
bufferOffset int64
|
2020-03-22 05:16:00 +00:00
|
|
|
bufferPos int
|
2020-03-21 03:31:11 +00:00
|
|
|
chunkIndex int
|
2020-03-22 08:00:36 +00:00
|
|
|
lookupFileId func(fileId string) (targetUrl string, err error)
|
2020-03-21 03:31:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var _ = io.ReadSeeker(&ChunkStreamReader{})
|
|
|
|
|
2020-03-22 08:00:36 +00:00
|
|
|
func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
|
2020-03-21 03:31:11 +00:00
|
|
|
|
2020-03-27 11:35:31 +00:00
|
|
|
chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32)
|
2020-03-21 03:31:11 +00:00
|
|
|
|
|
|
|
return &ChunkStreamReader{
|
2020-03-22 08:00:36 +00:00
|
|
|
chunkViews: chunkViews,
|
|
|
|
lookupFileId: func(fileId string) (targetUrl string, err error) {
|
|
|
|
return masterClient.LookupFileId(fileId)
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-21 03:31:11 +00:00
|
|
|
func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
|
2020-03-22 05:16:00 +00:00
|
|
|
if c.isBufferEmpty() {
|
2020-03-21 03:31:11 +00:00
|
|
|
if c.chunkIndex >= len(c.chunkViews) {
|
|
|
|
return 0, io.EOF
|
|
|
|
}
|
|
|
|
chunkView := c.chunkViews[c.chunkIndex]
|
|
|
|
c.fetchChunkToBuffer(chunkView)
|
|
|
|
c.chunkIndex++
|
|
|
|
}
|
2020-03-22 05:16:00 +00:00
|
|
|
n = copy(p, c.buffer[c.bufferPos:])
|
|
|
|
c.bufferPos += n
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *ChunkStreamReader) isBufferEmpty() bool {
|
|
|
|
return len(c.buffer) <= c.bufferPos
|
2020-03-21 03:31:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
|
2020-03-22 05:16:00 +00:00
|
|
|
|
2020-03-22 06:48:11 +00:00
|
|
|
var totalSize int64
|
|
|
|
for _, chunk := range c.chunkViews {
|
|
|
|
totalSize += int64(chunk.Size)
|
|
|
|
}
|
|
|
|
|
2020-03-22 05:16:00 +00:00
|
|
|
var err error
|
|
|
|
switch whence {
|
|
|
|
case io.SeekStart:
|
|
|
|
case io.SeekCurrent:
|
|
|
|
offset += c.bufferOffset + int64(c.bufferPos)
|
|
|
|
case io.SeekEnd:
|
2020-03-22 06:48:11 +00:00
|
|
|
offset = totalSize + offset
|
2020-03-22 05:16:00 +00:00
|
|
|
}
|
2020-03-22 06:48:11 +00:00
|
|
|
if offset > totalSize {
|
2020-03-22 05:16:00 +00:00
|
|
|
err = io.ErrUnexpectedEOF
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, chunk := range c.chunkViews {
|
|
|
|
if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
|
2020-03-22 08:00:36 +00:00
|
|
|
if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
|
2020-03-22 05:16:00 +00:00
|
|
|
c.fetchChunkToBuffer(chunk)
|
|
|
|
c.chunkIndex = i + 1
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
c.bufferPos = int(offset - c.bufferOffset)
|
|
|
|
|
|
|
|
return offset, err
|
|
|
|
|
2020-03-21 03:31:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
|
2020-03-22 08:00:36 +00:00
|
|
|
urlString, err := c.lookupFileId(chunkView.FileId)
|
2020-03-21 03:31:11 +00:00
|
|
|
if err != nil {
|
|
|
|
glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
|
|
|
|
return err
|
|
|
|
}
|
2020-03-22 05:16:00 +00:00
|
|
|
var buffer bytes.Buffer
|
2020-04-14 04:58:10 +00:00
|
|
|
err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
|
2020-03-22 05:16:00 +00:00
|
|
|
buffer.Write(data)
|
2020-03-21 03:31:11 +00:00
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
|
|
|
|
return err
|
|
|
|
}
|
2020-03-22 05:16:00 +00:00
|
|
|
c.buffer = buffer.Bytes()
|
|
|
|
c.bufferPos = 0
|
|
|
|
c.bufferOffset = chunkView.LogicOffset
|
|
|
|
|
2020-03-22 08:00:36 +00:00
|
|
|
// glog.V(0).Infof("read %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
|
|
|
|
|
2020-03-21 03:31:11 +00:00
|
|
|
return nil
|
|
|
|
}
|
2020-03-22 08:00:36 +00:00
|
|
|
|
2020-03-23 07:01:34 +00:00
|
|
|
func VolumeId(fileId string) string {
|
|
|
|
lastCommaIndex := strings.LastIndex(fileId, ",")
|
|
|
|
if lastCommaIndex > 0 {
|
|
|
|
return fileId[:lastCommaIndex]
|
2020-03-22 08:00:36 +00:00
|
|
|
}
|
2020-03-23 07:01:34 +00:00
|
|
|
return fileId
|
2020-03-22 08:00:36 +00:00
|
|
|
}
|