Browse Source
Squashed commit of the following:
Squashed commit of the following:
commit 465b47b8f7
Merge: f39272b54 9c440d472
Author: chrislu <chris.lu@gmail.com>
Date: Wed Jul 10 23:56:51 2024 -0700
Merge branch 'master' into merged-persisted-logs
commit f39272b5416bdd4d1714ce3e99aded0fe62e7d89
Author: chrislu <chris.lu@gmail.com>
Date: Wed Jul 10 23:49:26 2024 -0700
merge meta logs from multiple files
pull/5768/head
chrislu
5 months ago
3 changed files with 383 additions and 90 deletions
@ -0,0 +1,350 @@ |
|||
package filer |
|||
|
|||
import ( |
|||
"container/heap" |
|||
"context" |
|||
"fmt" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer" |
|||
"github.com/seaweedfs/seaweedfs/weed/wdclient" |
|||
"google.golang.org/protobuf/proto" |
|||
"io" |
|||
"math" |
|||
"strings" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/util" |
|||
) |
|||
|
|||
type LogFileEntry struct { |
|||
TsNs int64 |
|||
FileEntry *Entry |
|||
} |
|||
|
|||
func (f *Filer) collectPersistedLogBuffer(startPosition log_buffer.MessagePosition, stopTsNs int64) (v *OrderedLogVisitor, err error) { |
|||
|
|||
if stopTsNs != 0 && startPosition.Time.UnixNano() > stopTsNs { |
|||
return nil, io.EOF |
|||
} |
|||
|
|||
startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Year(), startPosition.Month(), startPosition.Day()) |
|||
|
|||
dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, math.MaxInt32, "", "", "") |
|||
if listDayErr != nil { |
|||
return nil, fmt.Errorf("fail to list log by day: %v", listDayErr) |
|||
} |
|||
|
|||
return NewOrderedLogVisitor(f, startPosition, stopTsNs, dayEntries) |
|||
|
|||
} |
|||
|
|||
// ----------
|
|||
type LogEntryItem struct { |
|||
Entry *filer_pb.LogEntry |
|||
filer string |
|||
} |
|||
|
|||
// LogEntryItemPriorityQueue a priority queue for LogEntry
|
|||
type LogEntryItemPriorityQueue []*LogEntryItem |
|||
|
|||
func (pq LogEntryItemPriorityQueue) Len() int { return len(pq) } |
|||
func (pq LogEntryItemPriorityQueue) Less(i, j int) bool { |
|||
return pq[i].Entry.TsNs < pq[j].Entry.TsNs |
|||
} |
|||
func (pq LogEntryItemPriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] } |
|||
func (pq *LogEntryItemPriorityQueue) Push(x any) { |
|||
item := x.(*LogEntryItem) |
|||
*pq = append(*pq, item) |
|||
} |
|||
func (pq *LogEntryItemPriorityQueue) Pop() any { |
|||
n := len(*pq) |
|||
item := (*pq)[n-1] |
|||
*pq = (*pq)[:n-1] |
|||
return item |
|||
} |
|||
|
|||
// ----------
|
|||
|
|||
type OrderedLogVisitor struct { |
|||
perFilerIteratorMap map[string]*LogFileQueueIterator |
|||
pq *LogEntryItemPriorityQueue |
|||
logFileEntryCollector *LogFileEntryCollector |
|||
} |
|||
|
|||
func NewOrderedLogVisitor(f *Filer, startPosition log_buffer.MessagePosition, stopTsNs int64, dayEntries []*Entry) (*OrderedLogVisitor, error) { |
|||
|
|||
perFilerQueueMap := make(map[string]*LogFileQueueIterator) |
|||
// initialize the priority queue
|
|||
pq := &LogEntryItemPriorityQueue{} |
|||
heap.Init(pq) |
|||
|
|||
t := &OrderedLogVisitor{ |
|||
perFilerIteratorMap: perFilerQueueMap, |
|||
pq: pq, |
|||
logFileEntryCollector: NewLogFileEntryCollector(f, startPosition, stopTsNs, dayEntries), |
|||
} |
|||
if err := t.logFileEntryCollector.collectMore(t); err != nil && err != io.EOF { |
|||
return nil, err |
|||
} |
|||
return t, nil |
|||
} |
|||
|
|||
func (o *OrderedLogVisitor) GetNext() (logEntry *filer_pb.LogEntry, err error) { |
|||
if o.pq.Len() == 0 { |
|||
return nil, io.EOF |
|||
} |
|||
item := heap.Pop(o.pq).(*LogEntryItem) |
|||
filerId := item.filer |
|||
|
|||
// fill the pq with the next log entry from the same filer
|
|||
it := o.perFilerIteratorMap[filerId] |
|||
next, nextErr := it.getNext(o) |
|||
if nextErr != nil { |
|||
if nextErr == io.EOF { |
|||
// do nothing since the filer has no more log entries
|
|||
}else { |
|||
return nil, fmt.Errorf("failed to get next log entry: %v", nextErr) |
|||
} |
|||
} else { |
|||
heap.Push(o.pq, &LogEntryItem{ |
|||
Entry: next, |
|||
filer: filerId, |
|||
}) |
|||
} |
|||
return item.Entry, nil |
|||
} |
|||
|
|||
func getFilerId(name string) string { |
|||
idx := strings.LastIndex(name, ".") |
|||
if idx < 0 { |
|||
return "" |
|||
} |
|||
return name[idx+1:] |
|||
} |
|||
|
|||
// ----------
|
|||
|
|||
type LogFileEntryCollector struct { |
|||
f *Filer |
|||
startTsNs int64 |
|||
stopTsNs int64 |
|||
dayEntryQueue *util.Queue[*Entry] |
|||
startDate string |
|||
startHourMinute string |
|||
stopDate string |
|||
stopHourMinute string |
|||
} |
|||
|
|||
func NewLogFileEntryCollector(f *Filer, startPosition log_buffer.MessagePosition, stopTsNs int64, dayEntries []*Entry) *LogFileEntryCollector { |
|||
dayEntryQueue := util.NewQueue[*Entry]() |
|||
for _, dayEntry := range dayEntries { |
|||
dayEntryQueue.Enqueue(dayEntry) |
|||
println("enqueue day entry", dayEntry.Name()) |
|||
} |
|||
|
|||
startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Year(), startPosition.Month(), startPosition.Day()) |
|||
startHourMinute := fmt.Sprintf("%02d-%02d", startPosition.Hour(), startPosition.Minute()) |
|||
var stopDate, stopHourMinute string |
|||
if stopTsNs != 0 { |
|||
stopTime := time.Unix(0, stopTsNs+24*60*60*int64(time.Nanosecond)).UTC() |
|||
stopDate = fmt.Sprintf("%04d-%02d-%02d", stopTime.Year(), stopTime.Month(), stopTime.Day()) |
|||
stopHourMinute = fmt.Sprintf("%02d-%02d", stopTime.Hour(), stopTime.Minute()) |
|||
} |
|||
|
|||
return &LogFileEntryCollector{ |
|||
f: f, |
|||
startTsNs: startPosition.UnixNano(), |
|||
stopTsNs: stopTsNs, |
|||
dayEntryQueue: dayEntryQueue, |
|||
startDate: startDate, |
|||
startHourMinute: startHourMinute, |
|||
stopDate: stopDate, |
|||
stopHourMinute: stopHourMinute, |
|||
} |
|||
} |
|||
|
|||
func (c *LogFileEntryCollector) hasMore() bool { |
|||
return c.dayEntryQueue.Len() > 0 |
|||
} |
|||
|
|||
func (c *LogFileEntryCollector) collectMore(v *OrderedLogVisitor) (err error) { |
|||
dayEntry := c.dayEntryQueue.Dequeue() |
|||
if dayEntry == nil { |
|||
return io.EOF |
|||
} |
|||
println("dequeue day entry", dayEntry.Name()) |
|||
if c.stopDate != "" { |
|||
if strings.Compare(dayEntry.Name(), c.stopDate) > 0 { |
|||
return io.EOF |
|||
} |
|||
} |
|||
|
|||
hourMinuteEntries, _, listHourMinuteErr := c.f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "") |
|||
if listHourMinuteErr != nil { |
|||
return fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) |
|||
} |
|||
freshFilerIds := make(map[string]string) |
|||
for _, hourMinuteEntry := range hourMinuteEntries { |
|||
// println("checking hh-mm", hourMinuteEntry.FullPath)
|
|||
hourMinute := util.FileNameBase(hourMinuteEntry.Name()) |
|||
if dayEntry.Name() == c.startDate { |
|||
if strings.Compare(hourMinute, c.startHourMinute) < 0 { |
|||
continue |
|||
} |
|||
} |
|||
if dayEntry.Name() == c.stopDate { |
|||
if strings.Compare(hourMinute, c.stopHourMinute) > 0 { |
|||
break |
|||
} |
|||
} |
|||
|
|||
tsMinute := fmt.Sprintf("%s-%s", dayEntry.Name(), hourMinute) |
|||
println(" enqueue", tsMinute) |
|||
t, parseErr := time.Parse("2006-01-02-15-04", tsMinute) |
|||
if parseErr != nil { |
|||
glog.Errorf("failed to parse %s: %v", tsMinute, parseErr) |
|||
continue |
|||
} |
|||
filerId := getFilerId(hourMinuteEntry.Name()) |
|||
iter, found := v.perFilerIteratorMap[filerId] |
|||
if !found { |
|||
iter = newLogFileQueueIterator(c.f.MasterClient, util.NewQueue[*LogFileEntry](), c.startTsNs, c.stopTsNs) |
|||
v.perFilerIteratorMap[filerId] = iter |
|||
freshFilerIds[filerId] = hourMinuteEntry.Name() |
|||
} |
|||
iter.q.Enqueue(&LogFileEntry{ |
|||
TsNs: t.UnixNano(), |
|||
FileEntry: hourMinuteEntry, |
|||
}) |
|||
} |
|||
|
|||
// fill the pq with the next log entry if it is a new filer
|
|||
for filerId, entryName := range freshFilerIds { |
|||
iter, found := v.perFilerIteratorMap[filerId] |
|||
if !found { |
|||
glog.Errorf("Unexpected! failed to find iterator for filer %s", filerId) |
|||
continue |
|||
} |
|||
next, err := iter.getNext(v) |
|||
if err != nil { |
|||
if err == io.EOF { |
|||
// do nothing since the filer has no more log entries
|
|||
} |
|||
return fmt.Errorf("failed to get next log entry for %v: %v", entryName, err) |
|||
} |
|||
heap.Push(v.pq, &LogEntryItem{ |
|||
Entry: next, |
|||
filer: filerId, |
|||
}) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// ----------
|
|||
|
|||
type LogFileQueueIterator struct { |
|||
q *util.Queue[*LogFileEntry] |
|||
masterClient *wdclient.MasterClient |
|||
startTsNs int64 |
|||
stopTsNs int64 |
|||
currentFileIterator *LogFileIterator |
|||
} |
|||
|
|||
func newLogFileQueueIterator(masterClient *wdclient.MasterClient, q *util.Queue[*LogFileEntry], startTsNs, stopTsNs int64) *LogFileQueueIterator { |
|||
return &LogFileQueueIterator{ |
|||
q: q, |
|||
masterClient: masterClient, |
|||
startTsNs: startTsNs, |
|||
stopTsNs: stopTsNs, |
|||
} |
|||
} |
|||
|
|||
// getNext will return io.EOF when done
|
|||
func (iter *LogFileQueueIterator) getNext(v *OrderedLogVisitor) (logEntry *filer_pb.LogEntry, err error) { |
|||
for { |
|||
if iter.currentFileIterator != nil { |
|||
logEntry, err = iter.currentFileIterator.getNext() |
|||
if err != io.EOF { |
|||
return |
|||
} |
|||
} |
|||
// now either iter.currentFileIterator is nil or err is io.EOF
|
|||
if iter.q.Len() == 0 { |
|||
return nil, io.EOF |
|||
} |
|||
t := iter.q.Dequeue() |
|||
if t == nil { |
|||
continue |
|||
} |
|||
// skip the file if it is after the stopTsNs
|
|||
if iter.stopTsNs != 0 && t.TsNs > iter.stopTsNs { |
|||
return nil, io.EOF |
|||
} |
|||
next := iter.q.Peek() |
|||
if next == nil { |
|||
if collectErr := v.logFileEntryCollector.collectMore(v); collectErr != nil && collectErr != io.EOF { |
|||
return nil, collectErr |
|||
} |
|||
} |
|||
// skip the file if the next entry is before the startTsNs
|
|||
if next != nil && next.TsNs <= iter.startTsNs { |
|||
continue |
|||
} |
|||
iter.currentFileIterator = newLogFileIterator(iter.masterClient, t.FileEntry, iter.startTsNs, iter.stopTsNs) |
|||
} |
|||
} |
|||
|
|||
// ----------
|
|||
|
|||
type LogFileIterator struct { |
|||
r io.Reader |
|||
sizeBuf []byte |
|||
startTsNs int64 |
|||
stopTsNs int64 |
|||
} |
|||
|
|||
func newLogFileIterator(masterClient *wdclient.MasterClient, fileEntry *Entry, startTsNs, stopTsNs int64) *LogFileIterator { |
|||
return &LogFileIterator{ |
|||
r: NewChunkStreamReaderFromFiler(masterClient, fileEntry.Chunks), |
|||
sizeBuf: make([]byte, 4), |
|||
startTsNs: startTsNs, |
|||
stopTsNs: stopTsNs, |
|||
} |
|||
} |
|||
|
|||
// getNext will return io.EOF when done
|
|||
func (iter *LogFileIterator) getNext() (logEntry *filer_pb.LogEntry, err error) { |
|||
var n int |
|||
for { |
|||
n, err = iter.r.Read(iter.sizeBuf) |
|||
if err != nil { |
|||
return |
|||
} |
|||
if n != 4 { |
|||
return nil, fmt.Errorf("size %d bytes, expected 4 bytes", n) |
|||
} |
|||
size := util.BytesToUint32(iter.sizeBuf) |
|||
// println("entry size", size)
|
|||
entryData := make([]byte, size) |
|||
n, err = iter.r.Read(entryData) |
|||
if err != nil { |
|||
return |
|||
} |
|||
if n != int(size) { |
|||
return nil, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size) |
|||
} |
|||
logEntry = &filer_pb.LogEntry{} |
|||
if err = proto.Unmarshal(entryData, logEntry); err != nil { |
|||
return |
|||
} |
|||
if logEntry.TsNs <= iter.startTsNs { |
|||
continue |
|||
} |
|||
if iter.stopTsNs != 0 && logEntry.TsNs > iter.stopTsNs { |
|||
return nil, io.EOF |
|||
} |
|||
return |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue