2018-05-13 07:11:26 +00:00
|
|
|
package filer2
|
|
|
|
|
2018-05-21 00:06:09 +00:00
|
|
|
import (
|
2018-09-09 23:26:11 +00:00
|
|
|
"fmt"
|
|
|
|
"hash/fnv"
|
2020-03-27 11:35:31 +00:00
|
|
|
"math"
|
2018-05-27 18:52:26 +00:00
|
|
|
"sort"
|
2018-11-18 19:51:38 +00:00
|
|
|
"sync"
|
2018-05-21 00:08:54 +00:00
|
|
|
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
2018-05-21 00:06:09 +00:00
|
|
|
)
|
2018-05-16 07:08:44 +00:00
|
|
|
|
2018-05-21 00:06:09 +00:00
|
|
|
func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
|
2018-05-13 07:11:26 +00:00
|
|
|
for _, c := range chunks {
|
|
|
|
t := uint64(c.Offset + int64(c.Size))
|
|
|
|
if size < t {
|
|
|
|
size = t
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-08-15 16:32:47 +00:00
|
|
|
func FileSize(entry *filer_pb.Entry) (size uint64) {
|
|
|
|
return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
|
|
|
|
}
|
|
|
|
|
2020-04-08 15:12:00 +00:00
|
|
|
func ETag(entry *filer_pb.Entry) (etag string) {
|
|
|
|
if entry.Attributes == nil || entry.Attributes.Md5 == nil {
|
2020-04-08 16:13:26 +00:00
|
|
|
return ETagChunks(entry.Chunks)
|
2020-04-08 15:12:00 +00:00
|
|
|
}
|
|
|
|
return fmt.Sprintf("%x", entry.Attributes.Md5)
|
|
|
|
}
|
|
|
|
|
|
|
|
func ETagEntry(entry *Entry) (etag string) {
|
|
|
|
if entry.Attr.Md5 == nil {
|
2020-04-08 16:13:26 +00:00
|
|
|
return ETagChunks(entry.Chunks)
|
2020-04-08 15:12:00 +00:00
|
|
|
}
|
|
|
|
return fmt.Sprintf("%x", entry.Attr.Md5)
|
|
|
|
}
|
|
|
|
|
|
|
|
func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
|
2018-09-09 23:25:43 +00:00
|
|
|
if len(chunks) == 1 {
|
|
|
|
return chunks[0].ETag
|
|
|
|
}
|
|
|
|
|
|
|
|
h := fnv.New32a()
|
|
|
|
for _, c := range chunks {
|
|
|
|
h.Write([]byte(c.ETag))
|
|
|
|
}
|
|
|
|
return fmt.Sprintf("%x", h.Sum32())
|
|
|
|
}
|
|
|
|
|
2020-07-20 00:59:43 +00:00
|
|
|
func CompactFileChunks(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
|
2018-05-21 07:00:28 +00:00
|
|
|
|
2020-07-20 00:59:43 +00:00
|
|
|
visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks)
|
2018-05-21 07:00:28 +00:00
|
|
|
|
|
|
|
fileIds := make(map[string]bool)
|
|
|
|
for _, interval := range visibles {
|
|
|
|
fileIds[interval.fileId] = true
|
|
|
|
}
|
|
|
|
for _, chunk := range chunks {
|
2019-06-23 03:04:56 +00:00
|
|
|
if _, found := fileIds[chunk.GetFileIdString()]; found {
|
2018-05-21 07:00:28 +00:00
|
|
|
compacted = append(compacted, chunk)
|
|
|
|
} else {
|
|
|
|
garbage = append(garbage, chunk)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-21 00:06:09 +00:00
|
|
|
return
|
2018-05-13 07:11:26 +00:00
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2020-07-20 00:59:43 +00:00
|
|
|
func MinusChunks(lookupFileIdFn LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error) {
|
|
|
|
|
|
|
|
aData, aMeta, aErr := ResolveChunkManifest(lookupFileIdFn, as)
|
|
|
|
if aErr != nil {
|
|
|
|
return nil, aErr
|
|
|
|
}
|
|
|
|
bData, bMeta, bErr := ResolveChunkManifest(lookupFileIdFn, bs)
|
|
|
|
if bErr != nil {
|
|
|
|
return nil, bErr
|
|
|
|
}
|
|
|
|
|
|
|
|
delta = append(delta, DoMinusChunks(aData, bData)...)
|
|
|
|
delta = append(delta, DoMinusChunks(aMeta, bMeta)...)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
|
2018-05-22 10:26:38 +00:00
|
|
|
|
|
|
|
fileIds := make(map[string]bool)
|
2019-06-23 03:04:56 +00:00
|
|
|
for _, interval := range bs {
|
|
|
|
fileIds[interval.GetFileIdString()] = true
|
2018-05-22 10:26:38 +00:00
|
|
|
}
|
2019-06-23 03:04:56 +00:00
|
|
|
for _, chunk := range as {
|
|
|
|
if _, found := fileIds[chunk.GetFileIdString()]; !found {
|
|
|
|
delta = append(delta, chunk)
|
2018-05-22 10:26:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-05-24 08:22:37 +00:00
|
|
|
type ChunkView struct {
|
|
|
|
FileId string
|
|
|
|
Offset int64
|
|
|
|
Size uint64
|
2020-08-16 22:16:46 +00:00
|
|
|
LogicOffset int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
|
2020-04-14 04:58:10 +00:00
|
|
|
ChunkSize uint64
|
2020-03-06 08:49:47 +00:00
|
|
|
CipherKey []byte
|
2020-03-28 20:42:35 +00:00
|
|
|
IsGzipped bool
|
2018-05-24 08:22:37 +00:00
|
|
|
}
|
|
|
|
|
2020-04-14 04:58:10 +00:00
|
|
|
func (cv *ChunkView) IsFullChunk() bool {
|
|
|
|
return cv.Size == cv.ChunkSize
|
|
|
|
}
|
|
|
|
|
2020-07-20 00:59:43 +00:00
|
|
|
func ViewFromChunks(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {
|
2018-05-24 05:28:54 +00:00
|
|
|
|
2020-07-20 00:59:43 +00:00
|
|
|
visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks)
|
2018-12-30 08:51:44 +00:00
|
|
|
|
|
|
|
return ViewFromVisibleIntervals(visibles, offset, size)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2020-03-22 08:37:46 +00:00
|
|
|
func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView) {
|
2018-05-24 05:28:54 +00:00
|
|
|
|
2020-03-22 08:37:46 +00:00
|
|
|
stop := offset + size
|
2020-03-27 11:35:31 +00:00
|
|
|
if size == math.MaxInt64 {
|
|
|
|
stop = math.MaxInt64
|
|
|
|
}
|
|
|
|
if stop < offset {
|
|
|
|
stop = math.MaxInt64
|
|
|
|
}
|
2018-05-24 05:28:54 +00:00
|
|
|
|
|
|
|
for _, chunk := range visibles {
|
2020-03-09 04:39:33 +00:00
|
|
|
|
2020-08-16 07:49:26 +00:00
|
|
|
chunkStart, chunkStop := max(offset, chunk.start), min(stop, chunk.stop)
|
|
|
|
|
|
|
|
if chunkStart < chunkStop {
|
2018-05-24 08:22:37 +00:00
|
|
|
views = append(views, &ChunkView{
|
|
|
|
FileId: chunk.fileId,
|
2020-08-16 22:16:46 +00:00
|
|
|
Offset: chunkStart - chunk.start + chunk.chunkOffset,
|
2020-08-16 07:49:26 +00:00
|
|
|
Size: uint64(chunkStop - chunkStart),
|
2020-08-16 08:35:52 +00:00
|
|
|
LogicOffset: chunkStart,
|
2020-04-14 04:58:10 +00:00
|
|
|
ChunkSize: chunk.chunkSize,
|
2020-03-06 08:49:47 +00:00
|
|
|
CipherKey: chunk.cipherKey,
|
2020-03-28 20:42:35 +00:00
|
|
|
IsGzipped: chunk.isGzipped,
|
2018-05-24 05:28:54 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return views
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2018-12-31 23:10:14 +00:00
|
|
|
func logPrintf(name string, visibles []VisibleInterval) {
|
2018-05-27 18:56:49 +00:00
|
|
|
/*
|
2018-11-23 08:26:15 +00:00
|
|
|
log.Printf("%s len %d", name, len(visibles))
|
|
|
|
for _, v := range visibles {
|
|
|
|
log.Printf("%s: => %+v", name, v)
|
|
|
|
}
|
2018-05-27 18:56:49 +00:00
|
|
|
*/
|
2018-05-13 07:11:26 +00:00
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2018-11-18 19:51:38 +00:00
|
|
|
var bufPool = sync.Pool{
|
|
|
|
New: func() interface{} {
|
2018-12-30 08:51:44 +00:00
|
|
|
return new(VisibleInterval)
|
2018-11-18 19:51:38 +00:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2019-01-06 06:23:44 +00:00
|
|
|
func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
|
2018-11-19 04:31:39 +00:00
|
|
|
|
2020-08-16 22:16:46 +00:00
|
|
|
newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, 0, chunk.Size, chunk.CipherKey, chunk.IsCompressed)
|
2018-11-19 04:31:39 +00:00
|
|
|
|
|
|
|
length := len(visibles)
|
|
|
|
if length == 0 {
|
|
|
|
return append(visibles, newV)
|
|
|
|
}
|
|
|
|
last := visibles[length-1]
|
|
|
|
if last.stop <= chunk.Offset {
|
|
|
|
return append(visibles, newV)
|
|
|
|
}
|
|
|
|
|
2018-11-22 00:25:13 +00:00
|
|
|
logPrintf(" before", visibles)
|
2020-08-16 22:16:46 +00:00
|
|
|
chunkStop := chunk.Offset + int64(chunk.Size)
|
2018-11-18 18:07:30 +00:00
|
|
|
for _, v := range visibles {
|
|
|
|
if v.start < chunk.Offset && chunk.Offset < v.stop {
|
2020-08-16 22:16:46 +00:00
|
|
|
newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, v.chunkOffset, v.chunkSize, v.cipherKey, v.isGzipped))
|
2018-05-21 00:06:09 +00:00
|
|
|
}
|
2018-11-18 18:07:30 +00:00
|
|
|
if v.start < chunkStop && chunkStop < v.stop {
|
2020-08-16 22:16:46 +00:00
|
|
|
newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, v.chunkOffset+(chunkStop-v.start), v.chunkSize, v.cipherKey, v.isGzipped))
|
2018-05-21 00:06:09 +00:00
|
|
|
}
|
2018-11-22 00:25:13 +00:00
|
|
|
if chunkStop <= v.start || v.stop <= chunk.Offset {
|
2018-11-18 18:07:30 +00:00
|
|
|
newVisibles = append(newVisibles, v)
|
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
}
|
2018-11-19 04:31:39 +00:00
|
|
|
newVisibles = append(newVisibles, newV)
|
|
|
|
|
2018-11-19 05:59:53 +00:00
|
|
|
logPrintf(" append", newVisibles)
|
|
|
|
|
|
|
|
for i := len(newVisibles) - 1; i >= 0; i-- {
|
|
|
|
if i > 0 && newV.start < newVisibles[i-1].start {
|
2018-11-19 04:31:39 +00:00
|
|
|
newVisibles[i] = newVisibles[i-1]
|
|
|
|
} else {
|
|
|
|
newVisibles[i] = newV
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
2018-11-19 05:59:53 +00:00
|
|
|
logPrintf(" sorted", newVisibles)
|
2018-11-19 04:31:39 +00:00
|
|
|
|
2018-11-19 05:24:58 +00:00
|
|
|
return newVisibles
|
2018-11-18 18:07:30 +00:00
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2020-07-20 00:59:43 +00:00
|
|
|
// NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
|
|
|
|
// If the file chunk content is a chunk manifest
|
|
|
|
func NonOverlappingVisibleIntervals(lookupFileIdFn LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (visibles []VisibleInterval, err error) {
|
|
|
|
|
|
|
|
chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks)
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2018-11-18 18:07:30 +00:00
|
|
|
sort.Slice(chunks, func(i, j int) bool {
|
|
|
|
return chunks[i].Mtime < chunks[j].Mtime
|
|
|
|
})
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2019-01-06 06:23:44 +00:00
|
|
|
var newVisibles []VisibleInterval
|
2018-11-18 18:07:30 +00:00
|
|
|
for _, chunk := range chunks {
|
2020-03-09 04:39:33 +00:00
|
|
|
|
2019-01-06 06:23:44 +00:00
|
|
|
newVisibles = MergeIntoVisibles(visibles, newVisibles, chunk)
|
2018-11-19 05:24:58 +00:00
|
|
|
t := visibles[:0]
|
2019-01-06 06:23:44 +00:00
|
|
|
visibles = newVisibles
|
|
|
|
newVisibles = t
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2018-11-19 05:59:53 +00:00
|
|
|
logPrintf("add", visibles)
|
|
|
|
|
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// find non-overlapping visible intervals
|
|
|
|
// visible interval map to one file chunk
|
|
|
|
|
2018-12-30 08:51:44 +00:00
|
|
|
type VisibleInterval struct {
|
2018-05-21 00:06:09 +00:00
|
|
|
start int64
|
|
|
|
stop int64
|
|
|
|
modifiedTime int64
|
|
|
|
fileId string
|
2020-08-16 22:16:46 +00:00
|
|
|
chunkOffset int64
|
2020-04-14 04:58:10 +00:00
|
|
|
chunkSize uint64
|
2020-03-06 08:49:47 +00:00
|
|
|
cipherKey []byte
|
2020-03-09 04:39:33 +00:00
|
|
|
isGzipped bool
|
2018-05-21 00:06:09 +00:00
|
|
|
}
|
|
|
|
|
2020-08-16 22:16:46 +00:00
|
|
|
func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, chunkOffset int64, chunkSize uint64, cipherKey []byte, isGzipped bool) VisibleInterval {
|
2018-12-31 23:10:14 +00:00
|
|
|
return VisibleInterval{
|
2018-11-19 04:31:39 +00:00
|
|
|
start: start,
|
|
|
|
stop: stop,
|
|
|
|
fileId: fileId,
|
|
|
|
modifiedTime: modifiedTime,
|
2020-08-16 22:16:46 +00:00
|
|
|
chunkOffset: chunkOffset, // the starting position in the chunk
|
2020-04-14 04:58:10 +00:00
|
|
|
chunkSize: chunkSize,
|
2020-03-06 08:49:47 +00:00
|
|
|
cipherKey: cipherKey,
|
2020-03-09 04:39:33 +00:00
|
|
|
isGzipped: isGzipped,
|
2018-11-19 04:31:39 +00:00
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func min(x, y int64) int64 {
|
|
|
|
if x <= y {
|
|
|
|
return x
|
|
|
|
}
|
|
|
|
return y
|
|
|
|
}
|
2020-08-16 07:49:26 +00:00
|
|
|
func max(x, y int64) int64 {
|
|
|
|
if x <= y {
|
|
|
|
return y
|
|
|
|
}
|
|
|
|
return x
|
|
|
|
}
|