2018-05-13 07:11:26 +00:00
|
|
|
package filer2
|
|
|
|
|
2018-05-21 00:06:09 +00:00
|
|
|
import (
|
|
|
|
"sort"
|
|
|
|
"log"
|
|
|
|
"math"
|
2018-05-21 00:08:54 +00:00
|
|
|
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
2018-05-21 00:06:09 +00:00
|
|
|
)
|
2018-05-16 07:08:44 +00:00
|
|
|
|
2018-05-21 00:06:09 +00:00
|
|
|
func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
|
2018-05-13 07:11:26 +00:00
|
|
|
for _, c := range chunks {
|
|
|
|
t := uint64(c.Offset + int64(c.Size))
|
|
|
|
if size < t {
|
|
|
|
size = t
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-05-21 00:06:09 +00:00
|
|
|
func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
|
2018-05-21 07:00:28 +00:00
|
|
|
|
|
|
|
visibles := nonOverlappingVisibleIntervals(chunks)
|
|
|
|
|
|
|
|
fileIds := make(map[string]bool)
|
|
|
|
for _, interval := range visibles {
|
|
|
|
fileIds[interval.fileId] = true
|
|
|
|
}
|
|
|
|
for _, chunk := range chunks {
|
|
|
|
if found := fileIds[chunk.FileId]; found {
|
|
|
|
compacted = append(compacted, chunk)
|
|
|
|
} else {
|
|
|
|
garbage = append(garbage, chunk)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-21 00:06:09 +00:00
|
|
|
return
|
2018-05-13 07:11:26 +00:00
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2018-05-21 00:08:54 +00:00
|
|
|
func logPrintf(name string, visibles []*visibleInterval) {
|
2018-05-21 00:06:09 +00:00
|
|
|
|
2018-05-21 07:00:28 +00:00
|
|
|
// return
|
2018-05-21 00:06:09 +00:00
|
|
|
|
|
|
|
log.Printf("%s len %d", name, len(visibles))
|
|
|
|
for _, v := range visibles {
|
|
|
|
log.Printf("%s: => %+v", name, v)
|
|
|
|
}
|
2018-05-13 07:11:26 +00:00
|
|
|
}
|
2018-05-21 00:06:09 +00:00
|
|
|
|
|
|
|
func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) {
|
|
|
|
|
|
|
|
sort.Slice(chunks, func(i, j int) bool {
|
|
|
|
if chunks[i].Offset < chunks[j].Offset {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
if chunks[i].Offset == chunks[j].Offset {
|
|
|
|
return chunks[i].Mtime < chunks[j].Mtime
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
})
|
|
|
|
|
|
|
|
if len(chunks) == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
var parallelIntervals, intervals []*visibleInterval
|
|
|
|
var minStopInterval, upToDateInterval *visibleInterval
|
|
|
|
watermarkStart := chunks[0].Offset
|
|
|
|
for _, chunk := range chunks {
|
2018-05-21 07:00:28 +00:00
|
|
|
log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
|
2018-05-21 00:06:09 +00:00
|
|
|
logPrintf("parallelIntervals", parallelIntervals)
|
|
|
|
for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset {
|
|
|
|
logPrintf("parallelIntervals loop 1", parallelIntervals)
|
|
|
|
logPrintf("parallelIntervals loop 1 intervals", intervals)
|
|
|
|
minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
|
|
|
|
nextStop := min(minStopInterval.stop, chunk.Offset)
|
|
|
|
intervals = append(intervals, newVisibleInterval(
|
|
|
|
max(watermarkStart, minStopInterval.start),
|
|
|
|
nextStop,
|
|
|
|
upToDateInterval.fileId,
|
|
|
|
upToDateInterval.modifiedTime,
|
|
|
|
))
|
|
|
|
watermarkStart = nextStop
|
|
|
|
logPrintf("parallelIntervals loop intervals =>", intervals)
|
|
|
|
|
|
|
|
// remove processed intervals, possibly multiple
|
|
|
|
var remaining []*visibleInterval
|
|
|
|
for _, interval := range parallelIntervals {
|
|
|
|
if interval.stop != watermarkStart {
|
2018-05-21 07:00:28 +00:00
|
|
|
remaining = append(remaining, interval)
|
2018-05-21 00:06:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
parallelIntervals = remaining
|
|
|
|
logPrintf("parallelIntervals loop 2", parallelIntervals)
|
|
|
|
logPrintf("parallelIntervals loop 2 intervals", intervals)
|
|
|
|
}
|
|
|
|
parallelIntervals = append(parallelIntervals, newVisibleInterval(
|
|
|
|
chunk.Offset,
|
|
|
|
chunk.Offset+int64(chunk.Size),
|
|
|
|
chunk.FileId,
|
|
|
|
chunk.Mtime,
|
|
|
|
))
|
|
|
|
}
|
|
|
|
|
|
|
|
logPrintf("parallelIntervals loop 3", parallelIntervals)
|
|
|
|
logPrintf("parallelIntervals loop 3 intervals", intervals)
|
|
|
|
for len(parallelIntervals) > 0 {
|
|
|
|
minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
|
|
|
|
intervals = append(intervals, newVisibleInterval(
|
|
|
|
max(watermarkStart, minStopInterval.start),
|
|
|
|
minStopInterval.stop,
|
|
|
|
upToDateInterval.fileId,
|
|
|
|
upToDateInterval.modifiedTime,
|
|
|
|
))
|
|
|
|
watermarkStart = minStopInterval.stop
|
|
|
|
|
|
|
|
// remove processed intervals, possibly multiple
|
|
|
|
var remaining []*visibleInterval
|
|
|
|
for _, interval := range parallelIntervals {
|
|
|
|
if interval.stop != watermarkStart {
|
2018-05-21 07:00:28 +00:00
|
|
|
remaining = append(remaining, interval)
|
2018-05-21 00:06:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
parallelIntervals = remaining
|
|
|
|
}
|
|
|
|
logPrintf("parallelIntervals loop 4", parallelIntervals)
|
|
|
|
logPrintf("intervals", intervals)
|
|
|
|
|
|
|
|
// merge connected intervals, now the intervals are non-intersecting
|
2018-05-21 07:00:28 +00:00
|
|
|
var lastIntervalIndex int
|
2018-05-21 00:06:09 +00:00
|
|
|
var prevIntervalIndex int
|
|
|
|
for i, interval := range intervals {
|
|
|
|
if i == 0 {
|
|
|
|
prevIntervalIndex = i
|
2018-05-21 07:00:28 +00:00
|
|
|
lastIntervalIndex = i
|
2018-05-21 00:06:09 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if intervals[i-1].fileId != interval.fileId ||
|
|
|
|
intervals[i-1].stop < intervals[i].start {
|
|
|
|
visibles = append(visibles, newVisibleInterval(
|
|
|
|
intervals[prevIntervalIndex].start,
|
|
|
|
intervals[i-1].stop,
|
|
|
|
intervals[prevIntervalIndex].fileId,
|
|
|
|
intervals[prevIntervalIndex].modifiedTime,
|
|
|
|
))
|
|
|
|
prevIntervalIndex = i
|
|
|
|
}
|
2018-05-21 07:00:28 +00:00
|
|
|
lastIntervalIndex = i
|
2018-05-21 00:06:09 +00:00
|
|
|
logPrintf("intervals loop 1 visibles", visibles)
|
|
|
|
}
|
|
|
|
|
2018-05-21 07:00:28 +00:00
|
|
|
visibles = append(visibles, newVisibleInterval(
|
|
|
|
intervals[prevIntervalIndex].start,
|
|
|
|
intervals[lastIntervalIndex].stop,
|
|
|
|
intervals[prevIntervalIndex].fileId,
|
|
|
|
intervals[prevIntervalIndex].modifiedTime,
|
|
|
|
))
|
2018-05-21 00:06:09 +00:00
|
|
|
|
|
|
|
logPrintf("visibles", visibles)
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func findMinStopInterval(intervals []*visibleInterval) (minStopInterval, upToDateInterval *visibleInterval) {
|
|
|
|
var latestMtime int64
|
|
|
|
latestIntervalIndex := 0
|
|
|
|
minStop := int64(math.MaxInt64)
|
|
|
|
minIntervalIndex := 0
|
|
|
|
for i, interval := range intervals {
|
|
|
|
if minStop > interval.stop {
|
|
|
|
minIntervalIndex = i
|
|
|
|
minStop = interval.stop
|
|
|
|
}
|
|
|
|
if latestMtime < interval.modifiedTime {
|
|
|
|
latestMtime = interval.modifiedTime
|
|
|
|
latestIntervalIndex = i
|
|
|
|
}
|
|
|
|
}
|
|
|
|
minStopInterval = intervals[minIntervalIndex]
|
|
|
|
upToDateInterval = intervals[latestIntervalIndex]
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// find non-overlapping visible intervals
|
|
|
|
// visible interval map to one file chunk
|
|
|
|
|
|
|
|
type visibleInterval struct {
|
|
|
|
start int64
|
|
|
|
stop int64
|
|
|
|
modifiedTime int64
|
|
|
|
fileId string
|
|
|
|
}
|
|
|
|
|
|
|
|
func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64) *visibleInterval {
|
|
|
|
return &visibleInterval{start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime}
|
|
|
|
}
|
|
|
|
|
|
|
|
func min(x, y int64) int64 {
|
|
|
|
if x <= y {
|
|
|
|
return x
|
|
|
|
}
|
|
|
|
return y
|
|
|
|
}
|
|
|
|
|
|
|
|
func max(x, y int64) int64 {
|
|
|
|
if x > y {
|
|
|
|
return x
|
|
|
|
}
|
|
|
|
return y
|
2018-05-13 07:11:26 +00:00
|
|
|
}
|