seaweedfs/weed/util/log_buffer/log_read.go

78 lines
1.8 KiB
Go
Raw Normal View History

package log_buffer
import (
"bytes"
"time"
"github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
func (logBuffer *LogBuffer) LoopProcessLogData(
2020-05-08 09:47:22 +00:00
startTreadTime time.Time,
waitForDataFn func() bool,
eachLogDataFn func(logEntry *filer_pb.LogEntry) error) (err error) {
// loop through all messages
var bytesBuf *bytes.Buffer
lastReadTime := startTreadTime
defer func() {
if bytesBuf != nil {
2020-08-27 05:40:15 +00:00
logBuffer.ReleaseMemory(bytesBuf)
}
}()
for {
if bytesBuf != nil {
2020-08-27 05:40:15 +00:00
logBuffer.ReleaseMemory(bytesBuf)
}
bytesBuf = logBuffer.ReadFromBuffer(lastReadTime)
2020-05-17 01:47:41 +00:00
// fmt.Printf("ReadFromBuffer by %v\n", lastReadTime)
if bytesBuf == nil {
if waitForDataFn() {
continue
} else {
return
}
}
buf := bytesBuf.Bytes()
2020-05-17 01:47:41 +00:00
// fmt.Printf("ReadFromBuffer by %v size %d\n", lastReadTime, len(buf))
batchSize := 0
var startReadTime time.Time
for pos := 0; pos+4 < len(buf); {
size := util.BytesToUint32(buf[pos : pos+4])
entryData := buf[pos+4 : pos+4+int(size)]
// fmt.Printf("read buffer read %d [%d,%d) from [0,%d)\n", batchSize, pos, pos+int(size)+4, len(buf))
logEntry := &filer_pb.LogEntry{}
if err = proto.Unmarshal(entryData, logEntry); err != nil {
glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err)
pos += 4 + int(size)
continue
}
lastReadTime = time.Unix(0, logEntry.TsNs)
if startReadTime.IsZero() {
startReadTime = lastReadTime
}
if err = eachLogDataFn(logEntry); err != nil {
return
}
pos += 4 + int(size)
batchSize++
}
// fmt.Printf("sent message ts[%d,%d] size %d\n", startReadTime.UnixNano(), lastReadTime.UnixNano(), batchSize)
}
2020-04-21 00:48:06 +00:00
}