You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
168 lines
3.5 KiB
168 lines
3.5 KiB
package queue
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/golang/protobuf/proto"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
|
)
|
|
|
|
type LogBuffer struct {
|
|
buf []byte
|
|
idx []int
|
|
pos int
|
|
startTime time.Time
|
|
stopTime time.Time
|
|
sizeBuf []byte
|
|
flushInterval time.Duration
|
|
flushFn func(startTime, stopTime time.Time, buf []byte)
|
|
notifyFn func()
|
|
isStopping bool
|
|
sync.RWMutex
|
|
}
|
|
|
|
func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte), notifyFn func()) *LogBuffer {
|
|
lb := &LogBuffer{
|
|
buf: make([]byte, 4*1024*1024),
|
|
sizeBuf: make([]byte, 4),
|
|
flushInterval: flushInterval,
|
|
flushFn: flushFn,
|
|
notifyFn: notifyFn,
|
|
}
|
|
go lb.loopFlush()
|
|
return lb
|
|
}
|
|
|
|
func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) {
|
|
|
|
logEntry := &filer_pb.LogEntry{
|
|
TsNs: ts.UnixNano(),
|
|
PartitionKeyHash: util.HashToInt32(key),
|
|
Data: data,
|
|
}
|
|
|
|
logEntryData, _ := proto.Marshal(logEntry)
|
|
|
|
size := len(logEntryData)
|
|
|
|
m.Lock()
|
|
defer func() {
|
|
m.Unlock()
|
|
if m.notifyFn != nil {
|
|
m.notifyFn()
|
|
}
|
|
}()
|
|
|
|
if m.pos == 0 {
|
|
m.startTime = ts
|
|
}
|
|
|
|
if m.startTime.Add(m.flushInterval).Before(ts) || len(m.buf)-m.pos < size+4 {
|
|
m.flush()
|
|
m.startTime = ts
|
|
if len(m.buf) < size+4 {
|
|
m.buf = make([]byte, 2*size+4)
|
|
}
|
|
}
|
|
m.stopTime = ts
|
|
|
|
m.idx = append(m.idx, m.pos)
|
|
util.Uint32toBytes(m.sizeBuf, uint32(size))
|
|
copy(m.buf[m.pos:m.pos+4], m.sizeBuf)
|
|
copy(m.buf[m.pos+4:m.pos+4+size], logEntryData)
|
|
m.pos += size + 4
|
|
}
|
|
|
|
func (m *LogBuffer) Shutdown() {
|
|
if m.isStopping {
|
|
return
|
|
}
|
|
m.isStopping = true
|
|
m.Lock()
|
|
m.flush()
|
|
m.Unlock()
|
|
}
|
|
|
|
func (m *LogBuffer) loopFlush() {
|
|
for !m.isStopping {
|
|
m.Lock()
|
|
m.flush()
|
|
m.Unlock()
|
|
time.Sleep(m.flushInterval)
|
|
}
|
|
}
|
|
|
|
func (m *LogBuffer) flush() {
|
|
|
|
if m.flushFn != nil && m.pos > 0 {
|
|
// fmt.Printf("flush buffer %d pos %d empty space %d\n", len(m.buf), m.pos, len(m.buf)-m.pos)
|
|
m.flushFn(m.startTime, m.stopTime, m.buf[:m.pos])
|
|
m.pos = 0
|
|
m.idx = m.idx[:0]
|
|
}
|
|
}
|
|
|
|
func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, bufferCopy []byte) {
|
|
m.RLock()
|
|
defer m.RUnlock()
|
|
|
|
// fmt.Printf("read from buffer: %v\n", lastReadTime)
|
|
|
|
if lastReadTime.Equal(m.stopTime) {
|
|
return lastReadTime, nil
|
|
}
|
|
if lastReadTime.After(m.stopTime) {
|
|
// glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime)
|
|
return lastReadTime, nil
|
|
}
|
|
if lastReadTime.Before(m.startTime) {
|
|
return m.stopTime, copiedBytes(m.buf[:m.pos])
|
|
}
|
|
|
|
lastTs := lastReadTime.UnixNano()
|
|
l, h := 0, len(m.idx)-1
|
|
|
|
// fmt.Printf("l=%d, h=%d\n", l, h)
|
|
for {
|
|
mid := (l + h) / 2
|
|
pos := m.idx[mid]
|
|
t := readTs(m.buf, m.idx[mid])
|
|
if t <= lastTs {
|
|
l = mid + 1
|
|
} else if lastTs < t {
|
|
var prevT int64
|
|
if mid > 0 {
|
|
prevT = readTs(m.buf, m.idx[mid-1])
|
|
}
|
|
if prevT <= lastTs {
|
|
return time.Unix(0, t), copiedBytes(m.buf[pos:m.pos])
|
|
}
|
|
h = mid - 1
|
|
}
|
|
// fmt.Printf("l=%d, h=%d\n", l, h)
|
|
}
|
|
|
|
}
|
|
func copiedBytes(buf []byte) (copied []byte) {
|
|
copied = make([]byte, len(buf))
|
|
copy(copied, buf)
|
|
return
|
|
}
|
|
|
|
func readTs(buf []byte, pos int) int64 {
|
|
|
|
size := util.BytesToUint32(buf[pos : pos+4])
|
|
entryData := buf[pos+4 : pos+4+int(size)]
|
|
logEntry := &filer_pb.LogEntry{}
|
|
|
|
err := proto.Unmarshal(entryData, logEntry)
|
|
if err != nil {
|
|
glog.Fatalf("unexpected unmarshal filer_pb.LogEntry: %v", err)
|
|
}
|
|
return logEntry.TsNs
|
|
|
|
}
|