|
|
@ -1,6 +1,7 @@ |
|
|
|
package sub_coordinator |
|
|
|
|
|
|
|
import ( |
|
|
|
"fmt" |
|
|
|
"sort" |
|
|
|
"sync" |
|
|
|
) |
|
|
@ -18,13 +19,14 @@ func NewInflightMessageTracker(capacity int) *InflightMessageTracker { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// InflightMessage tracks the message with the key and timestamp.
|
|
|
|
// EnflightMessage tracks the message with the key and timestamp.
|
|
|
|
// These messages are sent to the consumer group instances and waiting for ack.
|
|
|
|
func (imt *InflightMessageTracker) InflightMessage(key []byte, tsNs int64) { |
|
|
|
func (imt *InflightMessageTracker) EnflightMessage(key []byte, tsNs int64) { |
|
|
|
fmt.Printf("EnflightMessage(%s,%d)\n", string(key), tsNs) |
|
|
|
imt.mu.Lock() |
|
|
|
defer imt.mu.Unlock() |
|
|
|
imt.messages[string(key)] = tsNs |
|
|
|
imt.timestamps.Add(tsNs) |
|
|
|
imt.timestamps.EnflightTimestamp(tsNs) |
|
|
|
} |
|
|
|
|
|
|
|
// IsMessageAcknowledged returns true if the message has been acknowledged.
|
|
|
@ -35,7 +37,7 @@ func (imt *InflightMessageTracker) IsMessageAcknowledged(key []byte, tsNs int64) |
|
|
|
imt.mu.Lock() |
|
|
|
defer imt.mu.Unlock() |
|
|
|
|
|
|
|
if tsNs < imt.timestamps.Oldest() { |
|
|
|
if tsNs <= imt.timestamps.OldestAckedTimestamp() { |
|
|
|
return true |
|
|
|
} |
|
|
|
if tsNs > imt.timestamps.Latest() { |
|
|
@ -51,6 +53,7 @@ func (imt *InflightMessageTracker) IsMessageAcknowledged(key []byte, tsNs int64) |
|
|
|
|
|
|
|
// AcknowledgeMessage acknowledges the message with the key and timestamp.
|
|
|
|
func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bool { |
|
|
|
fmt.Printf("AcknowledgeMessage(%s,%d)\n", string(key), tsNs) |
|
|
|
imt.mu.Lock() |
|
|
|
defer imt.mu.Unlock() |
|
|
|
timestamp, exists := imt.messages[string(key)] |
|
|
@ -59,12 +62,12 @@ func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bo |
|
|
|
} |
|
|
|
delete(imt.messages, string(key)) |
|
|
|
// Remove the specific timestamp from the ring buffer.
|
|
|
|
imt.timestamps.Remove(tsNs) |
|
|
|
imt.timestamps.AckTimestamp(tsNs) |
|
|
|
return true |
|
|
|
} |
|
|
|
|
|
|
|
func (imt *InflightMessageTracker) GetOldest() int64 { |
|
|
|
return imt.timestamps.Oldest() |
|
|
|
func (imt *InflightMessageTracker) GetOldestAckedTimestamp() int64 { |
|
|
|
return imt.timestamps.OldestAckedTimestamp() |
|
|
|
} |
|
|
|
|
|
|
|
// IsInflight returns true if the message with the key is inflight.
|
|
|
@ -75,63 +78,81 @@ func (imt *InflightMessageTracker) IsInflight(key []byte) bool { |
|
|
|
return found |
|
|
|
} |
|
|
|
|
|
|
|
type TimestampStatus struct { |
|
|
|
Timestamp int64 |
|
|
|
Acked bool |
|
|
|
} |
|
|
|
|
|
|
|
// RingBuffer represents a circular buffer to hold timestamps.
|
|
|
|
type RingBuffer struct { |
|
|
|
buffer []int64 |
|
|
|
buffer []*TimestampStatus |
|
|
|
head int |
|
|
|
size int |
|
|
|
maxTimestamp int64 |
|
|
|
minAckedTs int64 |
|
|
|
} |
|
|
|
|
|
|
|
// NewRingBuffer creates a new RingBuffer of the given capacity.
|
|
|
|
func NewRingBuffer(capacity int) *RingBuffer { |
|
|
|
return &RingBuffer{ |
|
|
|
buffer: make([]int64, capacity), |
|
|
|
buffer: newBuffer(capacity), |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Add adds a new timestamp to the ring buffer.
|
|
|
|
func (rb *RingBuffer) Add(timestamp int64) { |
|
|
|
rb.buffer[rb.head] = timestamp |
|
|
|
rb.head = (rb.head + 1) % len(rb.buffer) |
|
|
|
func newBuffer(capacity int) []*TimestampStatus { |
|
|
|
buffer := make([]*TimestampStatus, capacity) |
|
|
|
for i := range buffer { |
|
|
|
buffer[i] = &TimestampStatus{} |
|
|
|
} |
|
|
|
return buffer |
|
|
|
} |
|
|
|
|
|
|
|
// EnflightTimestamp adds a new timestamp to the ring buffer.
|
|
|
|
func (rb *RingBuffer) EnflightTimestamp(timestamp int64) { |
|
|
|
if rb.size < len(rb.buffer) { |
|
|
|
rb.size++ |
|
|
|
} else { |
|
|
|
newBuf := newBuffer(2*len(rb.buffer)) |
|
|
|
for i := 0; i < rb.size; i++ { |
|
|
|
newBuf[i] = rb.buffer[(rb.head+len(rb.buffer)-rb.size+i)%len(rb.buffer)] |
|
|
|
} |
|
|
|
rb.buffer = newBuf |
|
|
|
rb.head = rb.size |
|
|
|
rb.size++ |
|
|
|
} |
|
|
|
head := rb.buffer[rb.head] |
|
|
|
head.Timestamp = timestamp |
|
|
|
head.Acked = false |
|
|
|
rb.head = (rb.head + 1) % len(rb.buffer) |
|
|
|
if timestamp > rb.maxTimestamp { |
|
|
|
rb.maxTimestamp = timestamp |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Remove removes the specified timestamp from the ring buffer.
|
|
|
|
func (rb *RingBuffer) Remove(timestamp int64) { |
|
|
|
// AckTimestamp removes the specified timestamp from the ring buffer.
|
|
|
|
func (rb *RingBuffer) AckTimestamp(timestamp int64) { |
|
|
|
// Perform binary search
|
|
|
|
index := sort.Search(rb.size, func(i int) bool { |
|
|
|
return rb.buffer[(rb.head+len(rb.buffer)-rb.size+i)%len(rb.buffer)] >= timestamp |
|
|
|
return rb.buffer[(rb.head+len(rb.buffer)-rb.size+i)%len(rb.buffer)].Timestamp >= timestamp |
|
|
|
}) |
|
|
|
actualIndex := (rb.head + len(rb.buffer) - rb.size + index) % len(rb.buffer) |
|
|
|
|
|
|
|
if index < rb.size && rb.buffer[actualIndex] == timestamp { |
|
|
|
// Shift elements to maintain the buffer order
|
|
|
|
for i := index; i < rb.size-1; i++ { |
|
|
|
fromIndex := (rb.head + len(rb.buffer) - rb.size + i + 1) % len(rb.buffer) |
|
|
|
toIndex := (rb.head + len(rb.buffer) - rb.size + i) % len(rb.buffer) |
|
|
|
rb.buffer[toIndex] = rb.buffer[fromIndex] |
|
|
|
} |
|
|
|
rb.buffer[actualIndex].Acked = true |
|
|
|
|
|
|
|
// Remove all the acknowledged timestamps from the buffer
|
|
|
|
startPos := (rb.head + len(rb.buffer) - rb.size) % len(rb.buffer) |
|
|
|
for i := 0; i < len(rb.buffer) && rb.buffer[(startPos+i)%len(rb.buffer)].Acked; i++ { |
|
|
|
rb.size-- |
|
|
|
rb.buffer[(rb.head+len(rb.buffer)-1)%len(rb.buffer)] = 0 // Clear the last element
|
|
|
|
rb.minAckedTs = rb.buffer[(startPos+i)%len(rb.buffer)].Timestamp |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Oldest returns the oldest timestamp in the ring buffer.
|
|
|
|
func (rb *RingBuffer) Oldest() int64 { |
|
|
|
if rb.size == 0 { |
|
|
|
return 0 |
|
|
|
} |
|
|
|
oldestIndex := (rb.head + len(rb.buffer) - rb.size) % len(rb.buffer) |
|
|
|
return rb.buffer[oldestIndex] |
|
|
|
// OldestAckedTimestamp returns the oldest that is already acked timestamp in the ring buffer.
|
|
|
|
func (rb *RingBuffer) OldestAckedTimestamp() int64 { |
|
|
|
return rb.minAckedTs |
|
|
|
} |
|
|
|
|
|
|
|
// Latest returns the most recently added timestamp in the ring buffer.
|
|
|
|
// Latest returns the most recently known timestamp in the ring buffer.
|
|
|
|
func (rb *RingBuffer) Latest() int64 { |
|
|
|
if rb.size == 0 { |
|
|
|
return 0 |
|
|
|
} |
|
|
|
latestIndex := (rb.head + len(rb.buffer) - 1) % len(rb.buffer) |
|
|
|
return rb.buffer[latestIndex] |
|
|
|
return rb.maxTimestamp |
|
|
|
} |