seaweedfs/weed/util/log_buffer/log_buffer.go

221 lines
4.9 KiB
Go
Raw Normal View History

2020-04-11 19:37:41 +00:00
package log_buffer
2020-03-31 05:57:45 +00:00
import (
"sync"
"time"
"github.com/golang/protobuf/proto"
2020-04-05 07:51:16 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2020-03-31 05:57:45 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
2020-04-11 09:19:45 +00:00
const BufferSize = 4 * 1024 * 1024
const PreviousBufferCount = 3
2020-04-05 23:51:30 +00:00
type dataToFlush struct {
startTime time.Time
stopTime time.Time
data []byte
}
2020-03-31 05:57:45 +00:00
type LogBuffer struct {
2020-04-11 09:19:45 +00:00
prevBuffers *SealedBuffers
2020-03-31 05:57:45 +00:00
buf []byte
2020-04-05 07:51:16 +00:00
idx []int
2020-03-31 05:57:45 +00:00
pos int
startTime time.Time
stopTime time.Time
sizeBuf []byte
flushInterval time.Duration
flushFn func(startTime, stopTime time.Time, buf []byte)
2020-04-05 07:51:16 +00:00
notifyFn func()
2020-03-31 05:57:45 +00:00
isStopping bool
2020-04-05 23:51:30 +00:00
flushChan chan *dataToFlush
2020-04-05 07:51:16 +00:00
sync.RWMutex
2020-03-31 05:57:45 +00:00
}
2020-04-05 07:51:16 +00:00
func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte), notifyFn func()) *LogBuffer {
2020-03-31 05:57:45 +00:00
lb := &LogBuffer{
2020-04-11 09:19:45 +00:00
prevBuffers: newSealedBuffers(PreviousBufferCount),
buf: make([]byte, BufferSize),
2020-03-31 05:57:45 +00:00
sizeBuf: make([]byte, 4),
flushInterval: flushInterval,
flushFn: flushFn,
2020-04-05 07:51:16 +00:00
notifyFn: notifyFn,
2020-04-05 23:51:30 +00:00
flushChan: make(chan *dataToFlush, 256),
2020-03-31 05:57:45 +00:00
}
go lb.loopFlush()
2020-04-05 23:51:30 +00:00
go lb.loopInterval()
2020-03-31 05:57:45 +00:00
return lb
}
2020-04-10 08:35:59 +00:00
func (m *LogBuffer) AddToBuffer(key, data []byte) {
2020-03-31 05:57:45 +00:00
2020-04-10 08:35:59 +00:00
m.Lock()
defer func() {
m.Unlock()
if m.notifyFn != nil {
m.notifyFn()
}
}()
// need to put the timestamp inside the lock
ts := time.Now()
2020-03-31 05:57:45 +00:00
logEntry := &filer_pb.LogEntry{
TsNs: ts.UnixNano(),
PartitionKeyHash: util.HashToInt32(key),
Data: data,
}
logEntryData, _ := proto.Marshal(logEntry)
size := len(logEntryData)
if m.pos == 0 {
m.startTime = ts
}
if m.startTime.Add(m.flushInterval).Before(ts) || len(m.buf)-m.pos < size+4 {
2020-04-05 23:51:30 +00:00
m.flushChan <- m.copyToFlush()
2020-03-31 05:57:45 +00:00
m.startTime = ts
2020-04-05 07:51:16 +00:00
if len(m.buf) < size+4 {
m.buf = make([]byte, 2*size+4)
}
2020-03-31 05:57:45 +00:00
}
m.stopTime = ts
2020-04-05 07:51:16 +00:00
m.idx = append(m.idx, m.pos)
2020-03-31 05:57:45 +00:00
util.Uint32toBytes(m.sizeBuf, uint32(size))
copy(m.buf[m.pos:m.pos+4], m.sizeBuf)
copy(m.buf[m.pos+4:m.pos+4+size], logEntryData)
m.pos += size + 4
}
func (m *LogBuffer) Shutdown() {
if m.isStopping {
return
}
m.isStopping = true
m.Lock()
2020-04-05 23:51:30 +00:00
toFlush := m.copyToFlush()
2020-03-31 05:57:45 +00:00
m.Unlock()
2020-04-05 23:51:30 +00:00
m.flushChan <- toFlush
close(m.flushChan)
2020-03-31 05:57:45 +00:00
}
func (m *LogBuffer) loopFlush() {
2020-04-05 23:51:30 +00:00
for d := range m.flushChan {
if d != nil {
m.flushFn(d.startTime, d.stopTime, d.data)
}
}
}
func (m *LogBuffer) loopInterval() {
2020-03-31 05:57:45 +00:00
for !m.isStopping {
m.Lock()
2020-04-05 23:51:30 +00:00
toFlush := m.copyToFlush()
2020-03-31 05:57:45 +00:00
m.Unlock()
2020-04-05 23:51:30 +00:00
m.flushChan <- toFlush
2020-03-31 05:57:45 +00:00
time.Sleep(m.flushInterval)
}
}
2020-04-05 23:51:30 +00:00
func (m *LogBuffer) copyToFlush() *dataToFlush {
2020-04-05 19:38:29 +00:00
2020-03-31 05:57:45 +00:00
if m.flushFn != nil && m.pos > 0 {
2020-04-06 03:15:17 +00:00
// fmt.Printf("flush buffer %d pos %d empty space %d\n", len(m.buf), m.pos, len(m.buf)-m.pos)
2020-04-05 23:51:30 +00:00
d := &dataToFlush{
startTime: m.startTime,
stopTime: m.stopTime,
data: copiedBytes(m.buf[:m.pos]),
}
2020-04-11 09:19:45 +00:00
m.buf = m.prevBuffers.SealBuffer(m.startTime, m.stopTime, m.buf)
2020-03-31 05:57:45 +00:00
m.pos = 0
2020-04-05 07:51:16 +00:00
m.idx = m.idx[:0]
2020-04-05 23:51:30 +00:00
return d
2020-04-05 07:51:16 +00:00
}
2020-04-05 23:51:30 +00:00
return nil
2020-04-05 07:51:16 +00:00
}
func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, bufferCopy []byte) {
m.RLock()
defer m.RUnlock()
// fmt.Printf("read from buffer: %v\n", lastReadTime)
if lastReadTime.Equal(m.stopTime) {
return lastReadTime, nil
}
if lastReadTime.After(m.stopTime) {
// glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime)
return lastReadTime, nil
2020-03-31 05:57:45 +00:00
}
2020-04-05 07:51:16 +00:00
if lastReadTime.Before(m.startTime) {
return m.stopTime, copiedBytes(m.buf[:m.pos])
}
lastTs := lastReadTime.UnixNano()
l, h := 0, len(m.idx)-1
2020-04-06 03:15:17 +00:00
2020-04-05 23:51:30 +00:00
/*
2020-04-10 08:35:59 +00:00
for i, pos := range m.idx {
logEntry, ts := readTs(m.buf, pos)
event := &filer_pb.FullEventNotification{}
proto.Unmarshal(logEntry.Data, event)
entry := event.EventNotification.OldEntry
if entry == nil {
entry = event.EventNotification.NewEntry
}
fmt.Printf("entry %d ts: %v offset:%d dir:%s name:%s\n", i, time.Unix(0, ts), pos, event.Directory, entry.Name)
2020-04-05 23:51:30 +00:00
}
2020-04-10 08:35:59 +00:00
fmt.Printf("l=%d, h=%d\n", l, h)
*/
2020-04-05 07:51:16 +00:00
2020-04-06 00:34:26 +00:00
for l <= h {
2020-04-05 07:51:16 +00:00
mid := (l + h) / 2
pos := m.idx[mid]
2020-04-06 03:15:17 +00:00
_, t := readTs(m.buf, m.idx[mid])
2020-04-05 07:51:16 +00:00
if t <= lastTs {
l = mid + 1
} else if lastTs < t {
var prevT int64
if mid > 0 {
2020-04-06 03:15:17 +00:00
_, prevT = readTs(m.buf, m.idx[mid-1])
2020-04-05 07:51:16 +00:00
}
if prevT <= lastTs {
2020-04-06 03:15:17 +00:00
// println("found mid = ", mid)
2020-04-05 07:51:16 +00:00
return time.Unix(0, t), copiedBytes(m.buf[pos:m.pos])
}
h = mid - 1
}
// fmt.Printf("l=%d, h=%d\n", l, h)
}
2020-04-06 03:15:17 +00:00
// FIXME: this could be that the buffer has been flushed already
// println("not found")
return lastReadTime, nil
2020-04-05 07:51:16 +00:00
}
func copiedBytes(buf []byte) (copied []byte) {
copied = make([]byte, len(buf))
copy(copied, buf)
return
}
2020-04-06 03:15:17 +00:00
func readTs(buf []byte, pos int) (*filer_pb.LogEntry, int64) {
2020-04-05 07:51:16 +00:00
size := util.BytesToUint32(buf[pos : pos+4])
entryData := buf[pos+4 : pos+4+int(size)]
logEntry := &filer_pb.LogEntry{}
err := proto.Unmarshal(entryData, logEntry)
if err != nil {
glog.Fatalf("unexpected unmarshal filer_pb.LogEntry: %v", err)
}
2020-04-06 03:15:17 +00:00
return logEntry, logEntry.TsNs
2020-04-05 07:51:16 +00:00
2020-03-31 05:57:45 +00:00
}