You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

220 lines
4.9 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package log_buffer
  2. import (
  3. "sync"
  4. "time"
  5. "github.com/golang/protobuf/proto"
  6. "github.com/chrislusf/seaweedfs/weed/glog"
  7. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  8. "github.com/chrislusf/seaweedfs/weed/util"
  9. )
  10. const BufferSize = 4 * 1024 * 1024
  11. const PreviousBufferCount = 3
  12. type dataToFlush struct {
  13. startTime time.Time
  14. stopTime time.Time
  15. data []byte
  16. }
  17. type LogBuffer struct {
  18. prevBuffers *SealedBuffers
  19. buf []byte
  20. idx []int
  21. pos int
  22. startTime time.Time
  23. stopTime time.Time
  24. sizeBuf []byte
  25. flushInterval time.Duration
  26. flushFn func(startTime, stopTime time.Time, buf []byte)
  27. notifyFn func()
  28. isStopping bool
  29. flushChan chan *dataToFlush
  30. sync.RWMutex
  31. }
  32. func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte), notifyFn func()) *LogBuffer {
  33. lb := &LogBuffer{
  34. prevBuffers: newSealedBuffers(PreviousBufferCount),
  35. buf: make([]byte, BufferSize),
  36. sizeBuf: make([]byte, 4),
  37. flushInterval: flushInterval,
  38. flushFn: flushFn,
  39. notifyFn: notifyFn,
  40. flushChan: make(chan *dataToFlush, 256),
  41. }
  42. go lb.loopFlush()
  43. go lb.loopInterval()
  44. return lb
  45. }
  46. func (m *LogBuffer) AddToBuffer(partitionKey, data []byte) {
  47. m.Lock()
  48. defer func() {
  49. m.Unlock()
  50. if m.notifyFn != nil {
  51. m.notifyFn()
  52. }
  53. }()
  54. // need to put the timestamp inside the lock
  55. ts := time.Now()
  56. logEntry := &filer_pb.LogEntry{
  57. TsNs: ts.UnixNano(),
  58. PartitionKeyHash: util.HashToInt32(partitionKey),
  59. Data: data,
  60. }
  61. logEntryData, _ := proto.Marshal(logEntry)
  62. size := len(logEntryData)
  63. if m.pos == 0 {
  64. m.startTime = ts
  65. }
  66. if m.startTime.Add(m.flushInterval).Before(ts) || len(m.buf)-m.pos < size+4 {
  67. m.flushChan <- m.copyToFlush()
  68. m.startTime = ts
  69. if len(m.buf) < size+4 {
  70. m.buf = make([]byte, 2*size+4)
  71. }
  72. }
  73. m.stopTime = ts
  74. m.idx = append(m.idx, m.pos)
  75. util.Uint32toBytes(m.sizeBuf, uint32(size))
  76. copy(m.buf[m.pos:m.pos+4], m.sizeBuf)
  77. copy(m.buf[m.pos+4:m.pos+4+size], logEntryData)
  78. m.pos += size + 4
  79. }
  80. func (m *LogBuffer) Shutdown() {
  81. if m.isStopping {
  82. return
  83. }
  84. m.isStopping = true
  85. m.Lock()
  86. toFlush := m.copyToFlush()
  87. m.Unlock()
  88. m.flushChan <- toFlush
  89. close(m.flushChan)
  90. }
  91. func (m *LogBuffer) loopFlush() {
  92. for d := range m.flushChan {
  93. if d != nil {
  94. m.flushFn(d.startTime, d.stopTime, d.data)
  95. }
  96. }
  97. }
  98. func (m *LogBuffer) loopInterval() {
  99. for !m.isStopping {
  100. m.Lock()
  101. toFlush := m.copyToFlush()
  102. m.Unlock()
  103. m.flushChan <- toFlush
  104. time.Sleep(m.flushInterval)
  105. }
  106. }
  107. func (m *LogBuffer) copyToFlush() *dataToFlush {
  108. if m.flushFn != nil && m.pos > 0 {
  109. // fmt.Printf("flush buffer %d pos %d empty space %d\n", len(m.buf), m.pos, len(m.buf)-m.pos)
  110. d := &dataToFlush{
  111. startTime: m.startTime,
  112. stopTime: m.stopTime,
  113. data: copiedBytes(m.buf[:m.pos]),
  114. }
  115. m.buf = m.prevBuffers.SealBuffer(m.startTime, m.stopTime, m.buf)
  116. m.pos = 0
  117. m.idx = m.idx[:0]
  118. return d
  119. }
  120. return nil
  121. }
  122. func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, bufferCopy []byte) {
  123. m.RLock()
  124. defer m.RUnlock()
  125. // fmt.Printf("read from buffer: %v\n", lastReadTime)
  126. if lastReadTime.Equal(m.stopTime) {
  127. return lastReadTime, nil
  128. }
  129. if lastReadTime.After(m.stopTime) {
  130. // glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime)
  131. return lastReadTime, nil
  132. }
  133. if lastReadTime.Before(m.startTime) {
  134. return m.stopTime, copiedBytes(m.buf[:m.pos])
  135. }
  136. lastTs := lastReadTime.UnixNano()
  137. l, h := 0, len(m.idx)-1
  138. /*
  139. for i, pos := range m.idx {
  140. logEntry, ts := readTs(m.buf, pos)
  141. event := &filer_pb.SubscribeMetadataResponse{}
  142. proto.Unmarshal(logEntry.Data, event)
  143. entry := event.EventNotification.OldEntry
  144. if entry == nil {
  145. entry = event.EventNotification.NewEntry
  146. }
  147. fmt.Printf("entry %d ts: %v offset:%d dir:%s name:%s\n", i, time.Unix(0, ts), pos, event.Directory, entry.Name)
  148. }
  149. fmt.Printf("l=%d, h=%d\n", l, h)
  150. */
  151. for l <= h {
  152. mid := (l + h) / 2
  153. pos := m.idx[mid]
  154. _, t := readTs(m.buf, m.idx[mid])
  155. if t <= lastTs {
  156. l = mid + 1
  157. } else if lastTs < t {
  158. var prevT int64
  159. if mid > 0 {
  160. _, prevT = readTs(m.buf, m.idx[mid-1])
  161. }
  162. if prevT <= lastTs {
  163. // println("found mid = ", mid)
  164. return time.Unix(0, t), copiedBytes(m.buf[pos:m.pos])
  165. }
  166. h = mid - 1
  167. }
  168. // fmt.Printf("l=%d, h=%d\n", l, h)
  169. }
  170. // FIXME: this could be that the buffer has been flushed already
  171. // println("not found")
  172. return lastReadTime, nil
  173. }
  174. func copiedBytes(buf []byte) (copied []byte) {
  175. copied = make([]byte, len(buf))
  176. copy(copied, buf)
  177. return
  178. }
  179. func readTs(buf []byte, pos int) (*filer_pb.LogEntry, int64) {
  180. size := util.BytesToUint32(buf[pos : pos+4])
  181. entryData := buf[pos+4 : pos+4+int(size)]
  182. logEntry := &filer_pb.LogEntry{}
  183. err := proto.Unmarshal(entryData, logEntry)
  184. if err != nil {
  185. glog.Fatalf("unexpected unmarshal filer_pb.LogEntry: %v", err)
  186. }
  187. return logEntry, logEntry.TsNs
  188. }