You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
92 lines
1.7 KiB
92 lines
1.7 KiB
package queue
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/golang/protobuf/proto"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
|
)
|
|
|
|
type LogBuffer struct {
|
|
buf []byte
|
|
pos int
|
|
startTime time.Time
|
|
stopTime time.Time
|
|
sizeBuf []byte
|
|
flushInterval time.Duration
|
|
flushFn func(startTime, stopTime time.Time, buf []byte)
|
|
isStopping bool
|
|
sync.Mutex
|
|
}
|
|
|
|
func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte)) *LogBuffer {
|
|
lb := &LogBuffer{
|
|
buf: make([]byte, 4*0124*1024),
|
|
sizeBuf: make([]byte, 4),
|
|
flushInterval: flushInterval,
|
|
flushFn: flushFn,
|
|
}
|
|
go lb.loopFlush()
|
|
return lb
|
|
}
|
|
|
|
func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) {
|
|
|
|
logEntry := &filer_pb.LogEntry{
|
|
TsNs: ts.UnixNano(),
|
|
PartitionKeyHash: util.HashToInt32(key),
|
|
Data: data,
|
|
}
|
|
|
|
logEntryData, _ := proto.Marshal(logEntry)
|
|
|
|
size := len(logEntryData)
|
|
|
|
m.Lock()
|
|
defer m.Unlock()
|
|
|
|
if m.pos == 0 {
|
|
m.startTime = ts
|
|
}
|
|
|
|
if m.startTime.Add(m.flushInterval).Before(ts) || len(m.buf)-m.pos < size+4 {
|
|
m.flush()
|
|
m.startTime = ts
|
|
}
|
|
m.stopTime = ts
|
|
|
|
util.Uint32toBytes(m.sizeBuf, uint32(size))
|
|
copy(m.buf[m.pos:m.pos+4], m.sizeBuf)
|
|
|
|
copy(m.buf[m.pos+4:m.pos+4+size], logEntryData)
|
|
m.pos += size + 4
|
|
}
|
|
|
|
func (m *LogBuffer) Shutdown() {
|
|
if m.isStopping {
|
|
return
|
|
}
|
|
m.isStopping = true
|
|
m.Lock()
|
|
m.flush()
|
|
m.Unlock()
|
|
}
|
|
|
|
func (m *LogBuffer) loopFlush() {
|
|
for !m.isStopping {
|
|
m.Lock()
|
|
m.flush()
|
|
m.Unlock()
|
|
time.Sleep(m.flushInterval)
|
|
}
|
|
}
|
|
|
|
func (m *LogBuffer) flush() {
|
|
if m.flushFn != nil && m.pos > 0 {
|
|
m.flushFn(m.startTime, m.stopTime, m.buf[:m.pos])
|
|
m.pos = 0
|
|
}
|
|
}
|