Browse Source

fmt

pull/7329/head
chrislu 5 months ago
parent
commit
0e481cf97a
  1. 30
      weed/util/log_buffer/log_read_stateless.go

30
weed/util/log_buffer/log_read_stateless.go

@ -138,28 +138,28 @@ func (logBuffer *LogBuffer) ReadMessagesAtOffset(startOffset int64, maxMessages
// Historical data - try to read from disk if ReadFromDiskFn is configured // Historical data - try to read from disk if ReadFromDiskFn is configured
glog.V(2).Infof("[StatelessRead] Requested offset %d < buffer start %d (historical data), attempting disk read", glog.V(2).Infof("[StatelessRead] Requested offset %d < buffer start %d (historical data), attempting disk read",
startOffset, bufferStartOffset) startOffset, bufferStartOffset)
// Check if disk read function is configured // Check if disk read function is configured
if logBuffer.ReadFromDiskFn == nil { if logBuffer.ReadFromDiskFn == nil {
glog.V(2).Infof("[StatelessRead] No disk read function configured, returning offset too old error") glog.V(2).Infof("[StatelessRead] No disk read function configured, returning offset too old error")
return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d too old (earliest in-memory: %d)", return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d too old (earliest in-memory: %d)",
startOffset, bufferStartOffset) startOffset, bufferStartOffset)
} }
// Read from disk (this is async/non-blocking if the ReadFromDiskFn is properly implemented) // Read from disk (this is async/non-blocking if the ReadFromDiskFn is properly implemented)
// The ReadFromDiskFn should handle its own timeouts and not block indefinitely // The ReadFromDiskFn should handle its own timeouts and not block indefinitely
diskMessages, diskNextOffset, diskErr := readHistoricalDataFromDisk( diskMessages, diskNextOffset, diskErr := readHistoricalDataFromDisk(
logBuffer, startOffset, maxMessages, maxBytes, highWaterMark) logBuffer, startOffset, maxMessages, maxBytes, highWaterMark)
if diskErr != nil { if diskErr != nil {
glog.V(2).Infof("[StatelessRead] Disk read failed for offset %d: %v", startOffset, diskErr) glog.V(2).Infof("[StatelessRead] Disk read failed for offset %d: %v", startOffset, diskErr)
return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d too old (earliest in-memory: %d): %v", return messages, startOffset, highWaterMark, false, fmt.Errorf("offset %d too old (earliest in-memory: %d): %v",
startOffset, bufferStartOffset, diskErr) startOffset, bufferStartOffset, diskErr)
} }
glog.V(2).Infof("[StatelessRead] Successfully read %d messages from disk, nextOffset=%d", glog.V(2).Infof("[StatelessRead] Successfully read %d messages from disk, nextOffset=%d",
len(diskMessages), diskNextOffset) len(diskMessages), diskNextOffset)
// Return disk data // Return disk data
endOfPartition = diskNextOffset >= bufferStartOffset && len(diskMessages) < maxMessages endOfPartition = diskNextOffset >= bufferStartOffset && len(diskMessages) < maxMessages
return diskMessages, diskNextOffset, highWaterMark, endOfPartition, nil return diskMessages, diskNextOffset, highWaterMark, endOfPartition, nil
@ -184,51 +184,51 @@ func readHistoricalDataFromDisk(
nextOffset = startOffset nextOffset = startOffset
totalBytes := 0 totalBytes := 0
messageCount := 0 messageCount := 0
// Create a position for the start offset // Create a position for the start offset
startPosition := MessagePosition{ startPosition := MessagePosition{
IsOffsetBased: true, IsOffsetBased: true,
Offset: startOffset, Offset: startOffset,
} }
// Define the callback function to collect messages // Define the callback function to collect messages
eachMessageFn := func(logEntry *filer_pb.LogEntry) (isDone bool, err error) { eachMessageFn := func(logEntry *filer_pb.LogEntry) (isDone bool, err error) {
// Check if we've reached the maxMessages or maxBytes limit // Check if we've reached the maxMessages or maxBytes limit
if messageCount >= maxMessages { if messageCount >= maxMessages {
return true, nil // Done, reached message limit return true, nil // Done, reached message limit
} }
entrySize := proto.Size(logEntry) entrySize := proto.Size(logEntry)
if totalBytes > 0 && totalBytes+entrySize > maxBytes { if totalBytes > 0 && totalBytes+entrySize > maxBytes {
return true, nil // Done, would exceed byte limit return true, nil // Done, would exceed byte limit
} }
// Add this message to the results // Add this message to the results
messages = append(messages, logEntry) messages = append(messages, logEntry)
messageCount++ messageCount++
totalBytes += entrySize totalBytes += entrySize
nextOffset++ nextOffset++
// Continue reading // Continue reading
return false, nil return false, nil
} }
// Call the ReadFromDiskFn to read historical data // Call the ReadFromDiskFn to read historical data
// This function should be non-blocking and have its own timeout handling // This function should be non-blocking and have its own timeout handling
lastPosition, isDone, readErr := logBuffer.ReadFromDiskFn(startPosition, 0, eachMessageFn) lastPosition, isDone, readErr := logBuffer.ReadFromDiskFn(startPosition, 0, eachMessageFn)
if readErr != nil { if readErr != nil {
return nil, startOffset, fmt.Errorf("failed to read from disk: %w", readErr) return nil, startOffset, fmt.Errorf("failed to read from disk: %w", readErr)
} }
// Update nextOffset based on what was actually read // Update nextOffset based on what was actually read
if lastPosition.IsOffsetBased { if lastPosition.IsOffsetBased {
nextOffset = lastPosition.Offset + 1 // Next offset to read nextOffset = lastPosition.Offset + 1 // Next offset to read
} }
glog.V(3).Infof("[DiskRead] Read %d messages from disk (offset %d to %d), isDone=%v", glog.V(3).Infof("[DiskRead] Read %d messages from disk (offset %d to %d), isDone=%v",
len(messages), startOffset, nextOffset-1, isDone) len(messages), startOffset, nextOffset-1, isDone)
return messages, nextOffset, nil return messages, nextOffset, nil
} }

Loading…
Cancel
Save