seaweedfs/weed/filer2/filechunks.go

228 lines
5.1 KiB
Go
Raw Normal View History

2018-05-13 07:11:26 +00:00
package filer2
2018-05-21 00:06:09 +00:00
import (
2018-09-09 23:26:11 +00:00
"fmt"
"hash/fnv"
2020-03-27 11:35:31 +00:00
"math"
2018-05-27 18:52:26 +00:00
"sort"
2018-11-18 19:51:38 +00:00
"sync"
2018-05-21 00:08:54 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
2018-05-21 00:06:09 +00:00
)
2018-05-21 00:06:09 +00:00
func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
2018-05-13 07:11:26 +00:00
for _, c := range chunks {
t := uint64(c.Offset + int64(c.Size))
if size < t {
size = t
}
}
return
}
2018-09-09 23:25:43 +00:00
func ETag(chunks []*filer_pb.FileChunk) (etag string) {
if len(chunks) == 1 {
return chunks[0].ETag
}
h := fnv.New32a()
for _, c := range chunks {
h.Write([]byte(c.ETag))
}
return fmt.Sprintf("%x", h.Sum32())
}
2018-05-21 00:06:09 +00:00
func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
visibles := NonOverlappingVisibleIntervals(chunks)
fileIds := make(map[string]bool)
for _, interval := range visibles {
fileIds[interval.fileId] = true
}
for _, chunk := range chunks {
2019-06-23 03:04:56 +00:00
if _, found := fileIds[chunk.GetFileIdString()]; found {
compacted = append(compacted, chunk)
} else {
garbage = append(garbage, chunk)
}
}
2018-05-21 00:06:09 +00:00
return
2018-05-13 07:11:26 +00:00
}
2018-05-21 00:06:09 +00:00
2019-06-23 03:04:56 +00:00
func MinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
fileIds := make(map[string]bool)
2019-06-23 03:04:56 +00:00
for _, interval := range bs {
fileIds[interval.GetFileIdString()] = true
}
2019-06-23 03:04:56 +00:00
for _, chunk := range as {
if _, found := fileIds[chunk.GetFileIdString()]; !found {
delta = append(delta, chunk)
}
}
return
}
type ChunkView struct {
FileId string
Offset int64
Size uint64
LogicOffset int64
IsFullChunk bool
CipherKey []byte
isGzipped bool
}
2020-03-22 08:37:46 +00:00
func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {
visibles := NonOverlappingVisibleIntervals(chunks)
return ViewFromVisibleIntervals(visibles, offset, size)
}
2020-03-22 08:37:46 +00:00
func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int64) (views []*ChunkView) {
2020-03-22 08:37:46 +00:00
stop := offset + size
2020-03-27 11:35:31 +00:00
if size == math.MaxInt64 {
stop = math.MaxInt64
}
if stop < offset {
stop = math.MaxInt64
}
for _, chunk := range visibles {
2018-05-25 06:19:56 +00:00
if chunk.start <= offset && offset < chunk.stop && offset < stop {
isFullChunk := chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop
views = append(views, &ChunkView{
FileId: chunk.fileId,
Offset: offset - chunk.start, // offset is the data starting location in this file id
Size: uint64(min(chunk.stop, stop) - offset),
LogicOffset: offset,
IsFullChunk: isFullChunk,
CipherKey: chunk.cipherKey,
isGzipped: chunk.isGzipped,
})
offset = min(chunk.stop, stop)
}
}
return views
}
2018-12-31 23:10:14 +00:00
func logPrintf(name string, visibles []VisibleInterval) {
2018-05-27 18:56:49 +00:00
/*
2018-11-23 08:26:15 +00:00
log.Printf("%s len %d", name, len(visibles))
for _, v := range visibles {
log.Printf("%s: => %+v", name, v)
}
2018-05-27 18:56:49 +00:00
*/
2018-05-13 07:11:26 +00:00
}
2018-05-21 00:06:09 +00:00
2018-11-18 19:51:38 +00:00
var bufPool = sync.Pool{
New: func() interface{} {
return new(VisibleInterval)
2018-11-18 19:51:38 +00:00
},
}
func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
2018-11-19 04:31:39 +00:00
newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey, chunk.IsGzipped)
2018-11-19 04:31:39 +00:00
length := len(visibles)
if length == 0 {
return append(visibles, newV)
}
last := visibles[length-1]
if last.stop <= chunk.Offset {
return append(visibles, newV)
}
logPrintf(" before", visibles)
2018-11-18 18:07:30 +00:00
for _, v := range visibles {
if v.start < chunk.Offset && chunk.Offset < v.stop {
newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped))
2018-05-21 00:06:09 +00:00
}
2018-11-18 18:07:30 +00:00
chunkStop := chunk.Offset + int64(chunk.Size)
if v.start < chunkStop && chunkStop < v.stop {
newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped))
2018-05-21 00:06:09 +00:00
}
if chunkStop <= v.start || v.stop <= chunk.Offset {
2018-11-18 18:07:30 +00:00
newVisibles = append(newVisibles, v)
}
2018-05-21 00:06:09 +00:00
}
2018-11-19 04:31:39 +00:00
newVisibles = append(newVisibles, newV)
2018-11-19 05:59:53 +00:00
logPrintf(" append", newVisibles)
for i := len(newVisibles) - 1; i >= 0; i-- {
if i > 0 && newV.start < newVisibles[i-1].start {
2018-11-19 04:31:39 +00:00
newVisibles[i] = newVisibles[i-1]
} else {
newVisibles[i] = newV
break
}
}
2018-11-19 05:59:53 +00:00
logPrintf(" sorted", newVisibles)
2018-11-19 04:31:39 +00:00
return newVisibles
2018-11-18 18:07:30 +00:00
}
2018-05-21 00:06:09 +00:00
2018-12-31 23:10:14 +00:00
func NonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
2018-05-21 00:06:09 +00:00
2018-11-18 18:07:30 +00:00
sort.Slice(chunks, func(i, j int) bool {
return chunks[i].Mtime < chunks[j].Mtime
})
2018-05-21 00:06:09 +00:00
var newVisibles []VisibleInterval
2018-11-18 18:07:30 +00:00
for _, chunk := range chunks {
newVisibles = MergeIntoVisibles(visibles, newVisibles, chunk)
t := visibles[:0]
visibles = newVisibles
newVisibles = t
2018-05-21 00:06:09 +00:00
2018-11-19 05:59:53 +00:00
logPrintf("add", visibles)
}
2018-05-21 00:06:09 +00:00
return
}
// find non-overlapping visible intervals
// visible interval map to one file chunk
type VisibleInterval struct {
2018-05-21 00:06:09 +00:00
start int64
stop int64
modifiedTime int64
fileId string
isFullChunk bool
cipherKey []byte
isGzipped bool
2018-05-21 00:06:09 +00:00
}
func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte, isGzipped bool) VisibleInterval {
2018-12-31 23:10:14 +00:00
return VisibleInterval{
2018-11-19 04:31:39 +00:00
start: start,
stop: stop,
fileId: fileId,
modifiedTime: modifiedTime,
isFullChunk: isFullChunk,
cipherKey: cipherKey,
isGzipped: isGzipped,
2018-11-19 04:31:39 +00:00
}
2018-05-21 00:06:09 +00:00
}
func min(x, y int64) int64 {
if x <= y {
return x
}
return y
}