diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go index 28ade51cc..ecb488373 100644 --- a/weed/filer2/filer_notify.go +++ b/weed/filer2/filer_notify.go @@ -133,6 +133,7 @@ func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func return fmt.Errorf("size %d bytes, expected 4 bytes", n) } size := util.BytesToUint32(sizeBuf) + // println("entry size", size) entryData := make([]byte, size) n, err = r.Read(entryData) if err != nil { diff --git a/weed/messaging/broker/broker_grpc_server_publish.go b/weed/messaging/broker/broker_grpc_server_publish.go index 985f708b5..b3a909a6c 100644 --- a/weed/messaging/broker/broker_grpc_server_publish.go +++ b/weed/messaging/broker/broker_grpc_server_publish.go @@ -22,7 +22,7 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis // TODO look it up topicConfig := &messaging_pb.TopicConfiguration{ - IsTransient: true, + // IsTransient: true, } // send init response diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index c5e033420..379063eed 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -34,7 +34,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs // TODO look it up topicConfig := &messaging_pb.TopicConfiguration{ - IsTransient: true, + // IsTransient: true, } if err = stream.Send(&messaging_pb.BrokerMessage{ @@ -79,7 +79,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err) return err } - // fmt.Printf("sending : %d bytes\n", len(m.Value)) + // fmt.Printf("sending : %d bytes ts %d\n", len(m.Value), logEntry.TsNs) if err = eachMessageFn(m); err != nil { glog.Errorf("sending %d bytes to %s: %s", len(m.Value), subscriberId, err) return err @@ -115,6 +115,7 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim startTsNs := startTime.UnixNano() topicDir := fmt.Sprintf("/topics/%s/%s", tp.Namespace, tp.Topic) + partitionSuffix := fmt.Sprintf(".part%02d", tp.Partition) return filer_pb.List(broker, topicDir, "", func(dayEntry *filer_pb.Entry, isLast bool) error { dayDir := fmt.Sprintf("%s/%s", topicDir, dayEntry.Name) @@ -124,7 +125,10 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim return nil } } - // println("processing", hourMinuteEntry.FullPath) + if !strings.HasSuffix(hourMinuteEntry.Name, partitionSuffix){ + return nil + } + // println("partition", tp.Partition, "processing", dayDir, "/", hourMinuteEntry.Name) chunkedFileReader := filer2.NewChunkStreamReader(broker, hourMinuteEntry.Chunks) defer chunkedFileReader.Close() if err := filer2.ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil { diff --git a/weed/messaging/broker/topic_lock.go b/weed/messaging/broker/topic_lock.go index d179537c9..f8a5aa171 100644 --- a/weed/messaging/broker/topic_lock.go +++ b/weed/messaging/broker/topic_lock.go @@ -42,7 +42,7 @@ func (locks *TopicLocks) buildLogBuffer(tl *TopicLock, tp TopicPartition, topicC flushFn := func(startTime, stopTime time.Time, buf []byte) { if topicConfig.IsTransient { - return + // return } // fmt.Printf("flushing with topic config %+v\n", topicConfig) diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go index 69d663484..6ba7f3737 100644 --- a/weed/util/log_buffer/log_buffer.go +++ b/weed/util/log_buffer/log_buffer.go @@ -93,7 +93,7 @@ func (m *LogBuffer) AddToBuffer(partitionKey, data []byte) { copy(m.buf[m.pos+4:m.pos+4+size], logEntryData) m.pos += size + 4 - // fmt.Printf("entry size %d total %d count %d\n", size, m.pos, len(m.idx)) + // fmt.Printf("entry size %d total %d count %d, buffer:%p\n", size, m.pos, len(m.idx), m) } @@ -112,6 +112,7 @@ func (m *LogBuffer) Shutdown() { func (m *LogBuffer) loopFlush() { for d := range m.flushChan { if d != nil { + // fmt.Printf("flush [%v, %v] size %d\n", d.startTime, d.stopTime, len(d.data.Bytes())) m.flushFn(d.startTime, d.stopTime, d.data.Bytes()) d.releaseMemory() } @@ -156,7 +157,12 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu m.RLock() defer m.RUnlock() - // fmt.Printf("read from buffer: %v last stop time: %v\n", lastReadTime.UnixNano(), m.stopTime.UnixNano()) + /* + fmt.Printf("read buffer %p: %v last stop time: [%v,%v], pos %d, entries:%d, prevBufs:%d\n", m, lastReadTime, m.startTime, m.stopTime, m.pos, len(m.idx), len(m.prevBuffers.buffers)) + for i, prevBuf := range m.prevBuffers.buffers { + fmt.Printf(" prev %d : %s\n", i, prevBuf.String()) + } + */ if lastReadTime.Equal(m.stopTime) { return nil @@ -173,6 +179,7 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu // println("return the earliest in memory", buf.startTime.UnixNano()) return copiedBytes(buf.buf[:buf.size]) } + // println("return the", i, "th in memory", buf.startTime.UnixNano()) return copiedBytes(buf.buf[:buf.size]) } if !buf.startTime.After(lastReadTime) && buf.stopTime.After(lastReadTime) { @@ -181,6 +188,7 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Bu return copiedBytes(buf.buf[pos:buf.size]) } } + // println("return the current buf", lastReadTime.UnixNano()) return copiedBytes(m.buf[:m.pos]) }