more efficient readResolvedChunks with linked list

weed/filer/filechunks_read.go

@@ -1,6 +1,7 @@
 package filer
 
 import (
+	"container/list"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"golang.org/x/exp/slices"
 )
@@ -33,40 +34,44 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterva
 	})
 
 	var prevX int64
-	var queue []*Point
+	queue := list.New() // points with higher ts are at the tail
+	var lastPoint *Point
 	for _, point := range points {
+		if queue.Len() > 0 {
+			lastPoint = queue.Back().Value.(*Point)
+		} else {
+			lastPoint = nil
+		}
 		if point.isStart {
-			if len(queue) > 0 {
+			if lastPoint != nil {
-				lastIndex := len(queue) - 1
-				lastPoint := queue[lastIndex]
 				if point.x != prevX && lastPoint.ts < point.ts {
 					visibles = addToVisibles(visibles, prevX, lastPoint, point)
 					prevX = point.x
 				}
 			}
 			// insert into queue
-			for i := len(queue); i >= 0; i-- {
+			if lastPoint == nil || lastPoint.ts < point.ts {
-				if i == 0 || queue[i-1].ts <= point.ts {
+				queue.PushBack(point)
-					if i == len(queue) {
 				prevX = point.x
-					}
+			} else {
-					queue = addToQueue(queue, i, point)
+				for e := queue.Front(); e != nil; e = e.Next() {
+					if e.Value.(*Point).ts > point.ts {
+						queue.InsertBefore(point, e)
 						break
 					}
 				}
+			}
 		} else {
-			lastIndex := len(queue) - 1
+			var isLast bool
-			index := lastIndex
+			for e := queue.Back(); e != nil; e = e.Prev() {
-			var startPoint *Point
+				isLast = e.Next() == nil
-			for ; index >= 0; index-- {
+				if e.Value.(*Point).ts == point.ts {
-				startPoint = queue[index]
+					queue.Remove(e)
-				if startPoint.ts == point.ts {
-					queue = removeFromQueue(queue, index)
 					break
 				}
 			}
-			if index == lastIndex && startPoint != nil {
+			if isLast && lastPoint != nil {
-				visibles = addToVisibles(visibles, prevX, startPoint, point)
+				visibles = addToVisibles(visibles, prevX, lastPoint, point)
 				prevX = point.x
 			}
 		}
@@ -75,22 +80,6 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterva
 	return
 }
 
-func removeFromQueue(queue []*Point, index int) []*Point {
-	for i := index; i < len(queue)-1; i++ {
-		queue[i] = queue[i+1]
-	}
-	queue = queue[:len(queue)-1]
-	return queue
-}
-
-func addToQueue(queue []*Point, index int, point *Point) []*Point {
-	queue = append(queue, point)
-	for i := len(queue) - 1; i > index; i-- {
-		queue[i], queue[i-1] = queue[i-1], queue[i]
-	}
-	return queue
-}
-
 func addToVisibles(visibles []VisibleInterval, prevX int64, startPoint *Point, point *Point) []VisibleInterval {
 	if prevX < point.x {
 		chunk := startPoint.chunk

weed/filer/filechunks_read_test.go

@@ -44,6 +44,33 @@ func TestReadResolvedChunks(t *testing.T) {
 
 	visibles := readResolvedChunks(chunks)
 
+	fmt.Printf("resolved to %d visible intervales\n", len(visibles))
+	for _, visible := range visibles {
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
+	}
+
+}
+
+func TestReadResolvedChunks2(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId:       "c",
+			Offset:       200,
+			Size:         50,
+			ModifiedTsNs: 3,
+		},
+		{
+			FileId:       "e",
+			Offset:       200,
+			Size:         25,
+			ModifiedTsNs: 5,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks)
+
+	fmt.Printf("resolved to %d visible intervales\n", len(visibles))
 	for _, visible := range visibles {
 		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
 	}