Browse Source
merge current message queue code changes (#6201)
merge current message queue code changes (#6201)
* listing files to convert to parquet * write parquet files * save logs into parquet files * pass by value * compact logs into parquet format * can skip existing files * refactor * refactor * fix compilation * when no partition found * refactor * add untested parquet file read * rename package * refactor * rename files * remove unused * add merged log read func * parquet wants to know the file size * rewind by time * pass in stop ts * add stop ts * adjust log * minor * adjust log * skip .parquet files when reading message logs * skip non message files * Update subscriber_record.go * send messages * skip message data with only ts * skip non log files * update parquet-go package * ensure a valid record type * add new field to a record type * Update read_parquet_to_log.go * fix parquet file name generation * separating reading parquet and logs * add key field * add skipped logs * use in memory cache * refactor * refactor * refactor * refactor, and change compact log * refactor * rename * refactor * fix format * prefix v to version directorypull/6209/head
Chris Lu
2 months ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 1106 additions and 264 deletions
-
2go.mod
-
2go.sum
-
4weed/command/benchmark.go
-
18weed/command/upload.go
-
4weed/filer/reader_at.go
-
5weed/filer/topics.go
-
36weed/filer_client/filer_client_accessor.go
-
3weed/mq/broker/broker_grpc_assign.go
-
2weed/mq/broker/broker_grpc_configure.go
-
5weed/mq/broker/broker_grpc_pub_follow.go
-
9weed/mq/broker/broker_grpc_sub_follow.go
-
5weed/mq/broker/broker_topic_conf_read_write.go
-
137weed/mq/broker/broker_topic_partition_read_write.go
-
13weed/mq/client/cmd/weed_pub_record/publisher_record.go
-
15weed/mq/client/cmd/weed_sub_record/subscriber_record.go
-
12weed/mq/client/sub_client/on_each_partition.go
-
1weed/mq/client/sub_client/subscriber.go
-
454weed/mq/logstore/log_to_parquet.go
-
41weed/mq/logstore/merged_read.go
-
144weed/mq/logstore/read_log_from_disk.go
-
162weed/mq/logstore/read_parquet_to_log.go
-
4weed/mq/pub_balancer/broker_stats.go
-
7weed/mq/schema/schema_builder.go
-
10weed/mq/sub_coordinator/partition_consumer_mapping.go
-
14weed/mq/sub_coordinator/partition_list.go
-
2weed/mq/topic/local_manager.go
-
1weed/mq/topic/local_partition.go
-
18weed/mq/topic/partition.go
-
44weed/mq/topic/topic.go
-
2weed/mq/topic/topic_partition.go
-
80weed/operation/submit.go
-
94weed/shell/command_mq_topic_compact.go
-
20weed/util/chunk_cache/chunk_cache_in_memory.go
@ -1,6 +1,7 @@ |
|||||
package filer |
package filer |
||||
|
|
||||
const ( |
const ( |
||||
TopicsDir = "/topics" |
|
||||
SystemLogDir = TopicsDir + "/.system/log" |
|
||||
|
TopicsDir = "/topics" |
||||
|
SystemLogDir = TopicsDir + "/.system/log" |
||||
|
TopicConfFile = "topic.conf" |
||||
) |
) |
||||
@ -0,0 +1,454 @@ |
|||||
|
package logstore |
||||
|
|
||||
|
import ( |
||||
|
"encoding/binary" |
||||
|
"fmt" |
||||
|
"github.com/parquet-go/parquet-go" |
||||
|
"github.com/parquet-go/parquet-go/compress/zstd" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/filer" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/schema" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/operation" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util" |
||||
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer" |
||||
|
"google.golang.org/protobuf/proto" |
||||
|
"io" |
||||
|
"os" |
||||
|
"strings" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
SW_COLUMN_NAME_TS = "_ts_ns" |
||||
|
SW_COLUMN_NAME_KEY = "_key" |
||||
|
) |
||||
|
|
||||
|
func CompactTopicPartitions(filerClient filer_pb.FilerClient, t topic.Topic, timeAgo time.Duration, recordType *schema_pb.RecordType, preference *operation.StoragePreference) error { |
||||
|
// list the topic partition versions
|
||||
|
topicVersions, err := collectTopicVersions(filerClient, t, timeAgo) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("list topic files: %v", err) |
||||
|
} |
||||
|
|
||||
|
// compact the partitions
|
||||
|
for _, topicVersion := range topicVersions { |
||||
|
partitions, err := collectTopicVersionsPartitions(filerClient, t, topicVersion) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("list partitions %s/%s/%s: %v", t.Namespace, t.Name, topicVersion, err) |
||||
|
} |
||||
|
for _, partition := range partitions { |
||||
|
err := compactTopicPartition(filerClient, t, timeAgo, recordType, partition, preference) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("compact partition %s/%s/%s/%s: %v", t.Namespace, t.Name, topicVersion, partition, err) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func collectTopicVersions(filerClient filer_pb.FilerClient, t topic.Topic, timeAgo time.Duration) (partitionVersions []time.Time, err error) { |
||||
|
err = filer_pb.ReadDirAllEntries(filerClient, util.FullPath(t.Dir()), "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
t, err := topic.ParseTopicVersion(entry.Name) |
||||
|
if err != nil { |
||||
|
// skip non-partition directories
|
||||
|
return nil |
||||
|
} |
||||
|
if t.Unix() < time.Now().Unix()-int64(timeAgo/time.Second) { |
||||
|
partitionVersions = append(partitionVersions, t) |
||||
|
} |
||||
|
return nil |
||||
|
}) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func collectTopicVersionsPartitions(filerClient filer_pb.FilerClient, t topic.Topic, topicVersion time.Time) (partitions []topic.Partition, err error) { |
||||
|
version := topicVersion.Format(topic.PartitionGenerationFormat) |
||||
|
err = filer_pb.ReadDirAllEntries(filerClient, util.FullPath(t.Dir()).Child(version), "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
if !entry.IsDirectory { |
||||
|
return nil |
||||
|
} |
||||
|
start, stop := topic.ParsePartitionBoundary(entry.Name) |
||||
|
if start != stop { |
||||
|
partitions = append(partitions, topic.Partition{ |
||||
|
RangeStart: start, |
||||
|
RangeStop: stop, |
||||
|
RingSize: topic.PartitionCount, |
||||
|
UnixTimeNs: topicVersion.UnixNano(), |
||||
|
}) |
||||
|
} |
||||
|
return nil |
||||
|
}) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func compactTopicPartition(filerClient filer_pb.FilerClient, t topic.Topic, timeAgo time.Duration, recordType *schema_pb.RecordType, partition topic.Partition, preference *operation.StoragePreference) error { |
||||
|
partitionDir := topic.PartitionDir(t, partition) |
||||
|
|
||||
|
// compact the partition directory
|
||||
|
return compactTopicPartitionDir(filerClient, t.Name, partitionDir, timeAgo, recordType, preference) |
||||
|
} |
||||
|
|
||||
|
func compactTopicPartitionDir(filerClient filer_pb.FilerClient, topicName, partitionDir string, timeAgo time.Duration, recordType *schema_pb.RecordType, preference *operation.StoragePreference) error { |
||||
|
// read all existing parquet files
|
||||
|
minTsNs, maxTsNs, err := readAllParquetFiles(filerClient, partitionDir) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// read all log files
|
||||
|
logFiles, err := readAllLogFiles(filerClient, partitionDir, timeAgo, minTsNs, maxTsNs) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
if len(logFiles) == 0 { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// divide log files into groups of 128MB
|
||||
|
logFileGroups := groupFilesBySize(logFiles, 128*1024*1024) |
||||
|
|
||||
|
// write to parquet file
|
||||
|
parquetLevels, err := schema.ToParquetLevels(recordType) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("ToParquetLevels failed %+v: %v", recordType, err) |
||||
|
} |
||||
|
|
||||
|
// create a parquet schema
|
||||
|
parquetSchema, err := schema.ToParquetSchema(topicName, recordType) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("ToParquetSchema failed: %v", err) |
||||
|
} |
||||
|
|
||||
|
// TODO parallelize the writing
|
||||
|
for _, logFileGroup := range logFileGroups { |
||||
|
if err = writeLogFilesToParquet(filerClient, partitionDir, recordType, logFileGroup, parquetSchema, parquetLevels, preference); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func groupFilesBySize(logFiles []*filer_pb.Entry, maxGroupSize int64) (logFileGroups [][]*filer_pb.Entry) { |
||||
|
var logFileGroup []*filer_pb.Entry |
||||
|
var groupSize int64 |
||||
|
for _, logFile := range logFiles { |
||||
|
if groupSize+int64(logFile.Attributes.FileSize) > maxGroupSize { |
||||
|
logFileGroups = append(logFileGroups, logFileGroup) |
||||
|
logFileGroup = nil |
||||
|
groupSize = 0 |
||||
|
} |
||||
|
logFileGroup = append(logFileGroup, logFile) |
||||
|
groupSize += int64(logFile.Attributes.FileSize) |
||||
|
} |
||||
|
if len(logFileGroup) > 0 { |
||||
|
logFileGroups = append(logFileGroups, logFileGroup) |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func readAllLogFiles(filerClient filer_pb.FilerClient, partitionDir string, timeAgo time.Duration, minTsNs, maxTsNs int64) (logFiles []*filer_pb.Entry, err error) { |
||||
|
err = filer_pb.ReadDirAllEntries(filerClient, util.FullPath(partitionDir), "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
if strings.HasSuffix(entry.Name, ".parquet") { |
||||
|
return nil |
||||
|
} |
||||
|
if entry.Attributes.Crtime > time.Now().Unix()-int64(timeAgo/time.Second) { |
||||
|
return nil |
||||
|
} |
||||
|
logTime, err := time.Parse(topic.TIME_FORMAT, entry.Name) |
||||
|
if err != nil { |
||||
|
// glog.Warningf("parse log time %s: %v", entry.Name, err)
|
||||
|
return nil |
||||
|
} |
||||
|
if maxTsNs > 0 && logTime.UnixNano() <= maxTsNs { |
||||
|
return nil |
||||
|
} |
||||
|
logFiles = append(logFiles, entry) |
||||
|
return nil |
||||
|
}) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func readAllParquetFiles(filerClient filer_pb.FilerClient, partitionDir string) (minTsNs, maxTsNs int64, err error) { |
||||
|
err = filer_pb.ReadDirAllEntries(filerClient, util.FullPath(partitionDir), "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
if !strings.HasSuffix(entry.Name, ".parquet") { |
||||
|
return nil |
||||
|
} |
||||
|
if len(entry.Extended) == 0 { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// read min ts
|
||||
|
minTsBytes := entry.Extended["min"] |
||||
|
if len(minTsBytes) != 8 { |
||||
|
return nil |
||||
|
} |
||||
|
minTs := int64(binary.BigEndian.Uint64(minTsBytes)) |
||||
|
if minTsNs == 0 || minTs < minTsNs { |
||||
|
minTsNs = minTs |
||||
|
} |
||||
|
|
||||
|
// read max ts
|
||||
|
maxTsBytes := entry.Extended["max"] |
||||
|
if len(maxTsBytes) != 8 { |
||||
|
return nil |
||||
|
} |
||||
|
maxTs := int64(binary.BigEndian.Uint64(maxTsBytes)) |
||||
|
if maxTsNs == 0 || maxTs > maxTsNs { |
||||
|
maxTsNs = maxTs |
||||
|
} |
||||
|
return nil |
||||
|
}) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func writeLogFilesToParquet(filerClient filer_pb.FilerClient, partitionDir string, recordType *schema_pb.RecordType, logFileGroups []*filer_pb.Entry, parquetSchema *parquet.Schema, parquetLevels *schema.ParquetLevels, preference *operation.StoragePreference) (err error) { |
||||
|
|
||||
|
tempFile, err := os.CreateTemp(".", "t*.parquet") |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("create temp file: %v", err) |
||||
|
} |
||||
|
defer func() { |
||||
|
tempFile.Close() |
||||
|
os.Remove(tempFile.Name()) |
||||
|
}() |
||||
|
|
||||
|
writer := parquet.NewWriter(tempFile, parquetSchema, parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel})) |
||||
|
rowBuilder := parquet.NewRowBuilder(parquetSchema) |
||||
|
|
||||
|
var startTsNs, stopTsNs int64 |
||||
|
|
||||
|
for _, logFile := range logFileGroups { |
||||
|
fmt.Printf("compact %s/%s ", partitionDir, logFile.Name) |
||||
|
var rows []parquet.Row |
||||
|
if err := iterateLogEntries(filerClient, logFile, func(entry *filer_pb.LogEntry) error { |
||||
|
|
||||
|
if startTsNs == 0 { |
||||
|
startTsNs = entry.TsNs |
||||
|
} |
||||
|
stopTsNs = entry.TsNs |
||||
|
|
||||
|
if len(entry.Key) == 0 { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// write to parquet file
|
||||
|
rowBuilder.Reset() |
||||
|
|
||||
|
record := &schema_pb.RecordValue{} |
||||
|
if err := proto.Unmarshal(entry.Data, record); err != nil { |
||||
|
return fmt.Errorf("unmarshal record value: %v", err) |
||||
|
} |
||||
|
|
||||
|
record.Fields[SW_COLUMN_NAME_TS] = &schema_pb.Value{ |
||||
|
Kind: &schema_pb.Value_Int64Value{ |
||||
|
Int64Value: entry.TsNs, |
||||
|
}, |
||||
|
} |
||||
|
record.Fields[SW_COLUMN_NAME_KEY] = &schema_pb.Value{ |
||||
|
Kind: &schema_pb.Value_BytesValue{ |
||||
|
BytesValue: entry.Key, |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
if err := schema.AddRecordValue(rowBuilder, recordType, parquetLevels, record); err != nil { |
||||
|
return fmt.Errorf("add record value: %v", err) |
||||
|
} |
||||
|
|
||||
|
rows = append(rows, rowBuilder.Row()) |
||||
|
|
||||
|
return nil |
||||
|
|
||||
|
}); err != nil { |
||||
|
return fmt.Errorf("iterate log entry %v/%v: %v", partitionDir, logFile.Name, err) |
||||
|
} |
||||
|
|
||||
|
fmt.Printf("processed %d rows\n", len(rows)) |
||||
|
|
||||
|
if _, err := writer.WriteRows(rows); err != nil { |
||||
|
return fmt.Errorf("write rows: %v", err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if err := writer.Close(); err != nil { |
||||
|
return fmt.Errorf("close writer: %v", err) |
||||
|
} |
||||
|
|
||||
|
// write to parquet file to partitionDir
|
||||
|
parquetFileName := fmt.Sprintf("%s.parquet", time.Unix(0, startTsNs).UTC().Format("2006-01-02-15-04-05")) |
||||
|
if err := saveParquetFileToPartitionDir(filerClient, tempFile, partitionDir, parquetFileName, preference, startTsNs, stopTsNs); err != nil { |
||||
|
return fmt.Errorf("save parquet file %s: %v", parquetFileName, err) |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func saveParquetFileToPartitionDir(filerClient filer_pb.FilerClient, sourceFile *os.File, partitionDir, parquetFileName string, preference *operation.StoragePreference, startTsNs, stopTsNs int64) error { |
||||
|
uploader, err := operation.NewUploader() |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("new uploader: %v", err) |
||||
|
} |
||||
|
|
||||
|
// get file size
|
||||
|
fileInfo, err := sourceFile.Stat() |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("stat source file: %v", err) |
||||
|
} |
||||
|
|
||||
|
// upload file in chunks
|
||||
|
chunkSize := int64(4 * 1024 * 1024) |
||||
|
chunkCount := (fileInfo.Size() + chunkSize - 1) / chunkSize |
||||
|
entry := &filer_pb.Entry{ |
||||
|
Name: parquetFileName, |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
Crtime: time.Now().Unix(), |
||||
|
Mtime: time.Now().Unix(), |
||||
|
FileMode: uint32(os.FileMode(0644)), |
||||
|
FileSize: uint64(fileInfo.Size()), |
||||
|
Mime: "application/vnd.apache.parquet", |
||||
|
}, |
||||
|
} |
||||
|
entry.Extended = make(map[string][]byte) |
||||
|
minTsBytes := make([]byte, 8) |
||||
|
binary.BigEndian.PutUint64(minTsBytes, uint64(startTsNs)) |
||||
|
entry.Extended["min"] = minTsBytes |
||||
|
maxTsBytes := make([]byte, 8) |
||||
|
binary.BigEndian.PutUint64(maxTsBytes, uint64(stopTsNs)) |
||||
|
entry.Extended["max"] = maxTsBytes |
||||
|
|
||||
|
for i := int64(0); i < chunkCount; i++ { |
||||
|
fileId, uploadResult, err, _ := uploader.UploadWithRetry( |
||||
|
filerClient, |
||||
|
&filer_pb.AssignVolumeRequest{ |
||||
|
Count: 1, |
||||
|
Replication: preference.Replication, |
||||
|
Collection: preference.Collection, |
||||
|
TtlSec: 0, // TODO set ttl
|
||||
|
DiskType: preference.DiskType, |
||||
|
Path: partitionDir + "/" + parquetFileName, |
||||
|
}, |
||||
|
&operation.UploadOption{ |
||||
|
Filename: parquetFileName, |
||||
|
Cipher: false, |
||||
|
IsInputCompressed: false, |
||||
|
MimeType: "application/vnd.apache.parquet", |
||||
|
PairMap: nil, |
||||
|
}, |
||||
|
func(host, fileId string) string { |
||||
|
return fmt.Sprintf("http://%s/%s", host, fileId) |
||||
|
}, |
||||
|
io.NewSectionReader(sourceFile, i*chunkSize, chunkSize), |
||||
|
) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("upload chunk %d: %v", i, err) |
||||
|
} |
||||
|
if uploadResult.Error != "" { |
||||
|
return fmt.Errorf("upload result: %v", uploadResult.Error) |
||||
|
} |
||||
|
entry.Chunks = append(entry.Chunks, uploadResult.ToPbFileChunk(fileId, i*chunkSize, time.Now().UnixNano())) |
||||
|
} |
||||
|
|
||||
|
// write the entry to partitionDir
|
||||
|
if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
return filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{ |
||||
|
Directory: partitionDir, |
||||
|
Entry: entry, |
||||
|
}) |
||||
|
}); err != nil { |
||||
|
return fmt.Errorf("create entry: %v", err) |
||||
|
} |
||||
|
fmt.Printf("saved to %s/%s\n", partitionDir, parquetFileName) |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func iterateLogEntries(filerClient filer_pb.FilerClient, logFile *filer_pb.Entry, eachLogEntryFn func(entry *filer_pb.LogEntry) error) error { |
||||
|
lookupFn := filer.LookupFn(filerClient) |
||||
|
_, err := eachFile(logFile, lookupFn, func(logEntry *filer_pb.LogEntry) (isDone bool, err error) { |
||||
|
if err := eachLogEntryFn(logEntry); err != nil { |
||||
|
return true, err |
||||
|
} |
||||
|
return false, nil |
||||
|
}) |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
func eachFile(entry *filer_pb.Entry, lookupFileIdFn func(fileId string) (targetUrls []string, err error), eachLogEntryFn log_buffer.EachLogEntryFuncType) (processedTsNs int64, err error) { |
||||
|
if len(entry.Content) > 0 { |
||||
|
// skip .offset files
|
||||
|
return |
||||
|
} |
||||
|
var urlStrings []string |
||||
|
for _, chunk := range entry.Chunks { |
||||
|
if chunk.Size == 0 { |
||||
|
continue |
||||
|
} |
||||
|
if chunk.IsChunkManifest { |
||||
|
fmt.Printf("this should not happen. unexpected chunk manifest in %s", entry.Name) |
||||
|
return |
||||
|
} |
||||
|
urlStrings, err = lookupFileIdFn(chunk.FileId) |
||||
|
if err != nil { |
||||
|
err = fmt.Errorf("lookup %s: %v", chunk.FileId, err) |
||||
|
return |
||||
|
} |
||||
|
if len(urlStrings) == 0 { |
||||
|
err = fmt.Errorf("no url found for %s", chunk.FileId) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// try one of the urlString until util.Get(urlString) succeeds
|
||||
|
var processed bool |
||||
|
for _, urlString := range urlStrings { |
||||
|
var data []byte |
||||
|
if data, _, err = util_http.Get(urlString); err == nil { |
||||
|
processed = true |
||||
|
if processedTsNs, err = eachChunk(data, eachLogEntryFn); err != nil { |
||||
|
return |
||||
|
} |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
if !processed { |
||||
|
err = fmt.Errorf("no data processed for %s %s", entry.Name, chunk.FileId) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func eachChunk(buf []byte, eachLogEntryFn log_buffer.EachLogEntryFuncType) (processedTsNs int64, err error) { |
||||
|
for pos := 0; pos+4 < len(buf); { |
||||
|
|
||||
|
size := util.BytesToUint32(buf[pos : pos+4]) |
||||
|
if pos+4+int(size) > len(buf) { |
||||
|
err = fmt.Errorf("reach each log chunk: read [%d,%d) from [0,%d)", pos, pos+int(size)+4, len(buf)) |
||||
|
return |
||||
|
} |
||||
|
entryData := buf[pos+4 : pos+4+int(size)] |
||||
|
|
||||
|
logEntry := &filer_pb.LogEntry{} |
||||
|
if err = proto.Unmarshal(entryData, logEntry); err != nil { |
||||
|
pos += 4 + int(size) |
||||
|
err = fmt.Errorf("unexpected unmarshal mq_pb.Message: %v", err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if _, err = eachLogEntryFn(logEntry); err != nil { |
||||
|
err = fmt.Errorf("process log entry %v: %v", logEntry, err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
processedTsNs = logEntry.TsNs |
||||
|
|
||||
|
pos += 4 + int(size) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
return |
||||
|
} |
||||
@ -0,0 +1,41 @@ |
|||||
|
package logstore |
||||
|
|
||||
|
import ( |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer" |
||||
|
) |
||||
|
|
||||
|
func GenMergedReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic.Partition) log_buffer.LogReadFromDiskFuncType { |
||||
|
fromParquetFn := GenParquetReadFunc(filerClient, t, p) |
||||
|
readLogDirectFn := GenLogOnDiskReadFunc(filerClient, t, p) |
||||
|
return mergeReadFuncs(fromParquetFn, readLogDirectFn) |
||||
|
} |
||||
|
|
||||
|
func mergeReadFuncs(fromParquetFn, readLogDirectFn log_buffer.LogReadFromDiskFuncType) log_buffer.LogReadFromDiskFuncType { |
||||
|
var exhaustedParquet bool |
||||
|
var lastProcessedPosition log_buffer.MessagePosition |
||||
|
return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastReadPosition log_buffer.MessagePosition, isDone bool, err error) { |
||||
|
if !exhaustedParquet { |
||||
|
// glog.V(4).Infof("reading from parquet startPosition: %v\n", startPosition.UTC())
|
||||
|
lastReadPosition, isDone, err = fromParquetFn(startPosition, stopTsNs, eachLogEntryFn) |
||||
|
// glog.V(4).Infof("read from parquet: %v %v %v %v\n", startPosition, lastReadPosition, isDone, err)
|
||||
|
if isDone { |
||||
|
isDone = false |
||||
|
} |
||||
|
if err != nil { |
||||
|
return |
||||
|
} |
||||
|
lastProcessedPosition = lastReadPosition |
||||
|
} |
||||
|
exhaustedParquet = true |
||||
|
|
||||
|
if startPosition.Before(lastProcessedPosition.Time) { |
||||
|
startPosition = lastProcessedPosition |
||||
|
} |
||||
|
|
||||
|
// glog.V(4).Infof("reading from direct log startPosition: %v\n", startPosition.UTC())
|
||||
|
lastReadPosition, isDone, err = readLogDirectFn(startPosition, stopTsNs, eachLogEntryFn) |
||||
|
return |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,144 @@ |
|||||
|
package logstore |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/filer" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/glog" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util" |
||||
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer" |
||||
|
"google.golang.org/protobuf/proto" |
||||
|
"math" |
||||
|
"strings" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
func GenLogOnDiskReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic.Partition) log_buffer.LogReadFromDiskFuncType { |
||||
|
partitionDir := topic.PartitionDir(t, p) |
||||
|
|
||||
|
lookupFileIdFn := filer.LookupFn(filerClient) |
||||
|
|
||||
|
eachChunkFn := func(buf []byte, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64) (processedTsNs int64, err error) { |
||||
|
for pos := 0; pos+4 < len(buf); { |
||||
|
|
||||
|
size := util.BytesToUint32(buf[pos : pos+4]) |
||||
|
if pos+4+int(size) > len(buf) { |
||||
|
err = fmt.Errorf("GenLogOnDiskReadFunc: read [%d,%d) from [0,%d)", pos, pos+int(size)+4, len(buf)) |
||||
|
return |
||||
|
} |
||||
|
entryData := buf[pos+4 : pos+4+int(size)] |
||||
|
|
||||
|
logEntry := &filer_pb.LogEntry{} |
||||
|
if err = proto.Unmarshal(entryData, logEntry); err != nil { |
||||
|
pos += 4 + int(size) |
||||
|
err = fmt.Errorf("unexpected unmarshal mq_pb.Message: %v", err) |
||||
|
return |
||||
|
} |
||||
|
if logEntry.TsNs < starTsNs { |
||||
|
pos += 4 + int(size) |
||||
|
continue |
||||
|
} |
||||
|
if stopTsNs != 0 && logEntry.TsNs > stopTsNs { |
||||
|
println("stopTsNs", stopTsNs, "logEntry.TsNs", logEntry.TsNs) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// fmt.Printf(" read logEntry: %v, ts %v\n", string(logEntry.Key), time.Unix(0, logEntry.TsNs).UTC())
|
||||
|
if _, err = eachLogEntryFn(logEntry); err != nil { |
||||
|
err = fmt.Errorf("process log entry %v: %v", logEntry, err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
processedTsNs = logEntry.TsNs |
||||
|
|
||||
|
pos += 4 + int(size) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
eachFileFn := func(entry *filer_pb.Entry, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64) (processedTsNs int64, err error) { |
||||
|
if len(entry.Content) > 0 { |
||||
|
// skip .offset files
|
||||
|
return |
||||
|
} |
||||
|
var urlStrings []string |
||||
|
for _, chunk := range entry.Chunks { |
||||
|
if chunk.Size == 0 { |
||||
|
continue |
||||
|
} |
||||
|
if chunk.IsChunkManifest { |
||||
|
glog.Warningf("this should not happen. unexpected chunk manifest in %s/%s", partitionDir, entry.Name) |
||||
|
return |
||||
|
} |
||||
|
urlStrings, err = lookupFileIdFn(chunk.FileId) |
||||
|
if err != nil { |
||||
|
err = fmt.Errorf("lookup %s: %v", chunk.FileId, err) |
||||
|
return |
||||
|
} |
||||
|
if len(urlStrings) == 0 { |
||||
|
err = fmt.Errorf("no url found for %s", chunk.FileId) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// try one of the urlString until util.Get(urlString) succeeds
|
||||
|
var processed bool |
||||
|
for _, urlString := range urlStrings { |
||||
|
// TODO optimization opportunity: reuse the buffer
|
||||
|
var data []byte |
||||
|
// fmt.Printf("reading %s/%s %s\n", partitionDir, entry.Name, urlString)
|
||||
|
if data, _, err = util_http.Get(urlString); err == nil { |
||||
|
processed = true |
||||
|
if processedTsNs, err = eachChunkFn(data, eachLogEntryFn, starTsNs, stopTsNs); err != nil { |
||||
|
return |
||||
|
} |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
if !processed { |
||||
|
err = fmt.Errorf("no data processed for %s %s", entry.Name, chunk.FileId) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastReadPosition log_buffer.MessagePosition, isDone bool, err error) { |
||||
|
startFileName := startPosition.UTC().Format(topic.TIME_FORMAT) |
||||
|
startTsNs := startPosition.Time.UnixNano() |
||||
|
stopTime := time.Unix(0, stopTsNs) |
||||
|
var processedTsNs int64 |
||||
|
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
return filer_pb.SeaweedList(client, partitionDir, "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
if entry.IsDirectory { |
||||
|
return nil |
||||
|
} |
||||
|
if strings.HasSuffix(entry.Name, ".parquet") { |
||||
|
return nil |
||||
|
} |
||||
|
// FIXME: this is a hack to skip the .offset files
|
||||
|
if strings.HasSuffix(entry.Name, ".offset") { |
||||
|
return nil |
||||
|
} |
||||
|
if stopTsNs != 0 && entry.Name > stopTime.UTC().Format(topic.TIME_FORMAT) { |
||||
|
isDone = true |
||||
|
return nil |
||||
|
} |
||||
|
if entry.Name < startPosition.UTC().Format(topic.TIME_FORMAT) { |
||||
|
return nil |
||||
|
} |
||||
|
if processedTsNs, err = eachFileFn(entry, eachLogEntryFn, startTsNs, stopTsNs); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
return nil |
||||
|
|
||||
|
}, startFileName, true, math.MaxInt32) |
||||
|
}) |
||||
|
lastReadPosition = log_buffer.NewMessagePosition(processedTsNs, -2) |
||||
|
return |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,162 @@ |
|||||
|
package logstore |
||||
|
|
||||
|
import ( |
||||
|
"encoding/binary" |
||||
|
"fmt" |
||||
|
"github.com/parquet-go/parquet-go" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/filer" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/schema" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer" |
||||
|
"google.golang.org/protobuf/proto" |
||||
|
"io" |
||||
|
"math" |
||||
|
"strings" |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
chunkCache = chunk_cache.NewChunkCacheInMemory(256) // 256 entries, 8MB max per entry
|
||||
|
) |
||||
|
|
||||
|
func GenParquetReadFunc(filerClient filer_pb.FilerClient, t topic.Topic, p topic.Partition) log_buffer.LogReadFromDiskFuncType { |
||||
|
partitionDir := topic.PartitionDir(t, p) |
||||
|
|
||||
|
lookupFileIdFn := filer.LookupFn(filerClient) |
||||
|
|
||||
|
// read topic conf from filer
|
||||
|
var topicConf *mq_pb.ConfigureTopicResponse |
||||
|
var err error |
||||
|
if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
topicConf, err = t.ReadConfFile(client) |
||||
|
return err |
||||
|
}); err != nil { |
||||
|
return nil |
||||
|
} |
||||
|
recordType := topicConf.GetRecordType() |
||||
|
recordType = schema.NewRecordTypeBuilder(recordType). |
||||
|
WithField(SW_COLUMN_NAME_TS, schema.TypeInt64). |
||||
|
WithField(SW_COLUMN_NAME_KEY, schema.TypeBytes). |
||||
|
RecordTypeEnd() |
||||
|
|
||||
|
parquetSchema, err := schema.ToParquetSchema(t.Name, recordType) |
||||
|
if err != nil { |
||||
|
return nil |
||||
|
} |
||||
|
parquetLevels, err := schema.ToParquetLevels(recordType) |
||||
|
if err != nil { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// eachFileFn reads a parquet file and calls eachLogEntryFn for each log entry
|
||||
|
eachFileFn := func(entry *filer_pb.Entry, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64) (processedTsNs int64, err error) { |
||||
|
// create readerAt for the parquet file
|
||||
|
fileSize := filer.FileSize(entry) |
||||
|
visibleIntervals, _ := filer.NonOverlappingVisibleIntervals(lookupFileIdFn, entry.Chunks, 0, int64(fileSize)) |
||||
|
chunkViews := filer.ViewFromVisibleIntervals(visibleIntervals, 0, int64(fileSize)) |
||||
|
readerCache := filer.NewReaderCache(32, chunkCache, lookupFileIdFn) |
||||
|
readerAt := filer.NewChunkReaderAtFromClient(readerCache, chunkViews, int64(fileSize)) |
||||
|
|
||||
|
// create parquet reader
|
||||
|
parquetReader := parquet.NewReader(readerAt, parquetSchema) |
||||
|
rows := make([]parquet.Row, 128) |
||||
|
for { |
||||
|
rowCount, readErr := parquetReader.ReadRows(rows) |
||||
|
|
||||
|
for i := 0; i < rowCount; i++ { |
||||
|
row := rows[i] |
||||
|
// convert parquet row to schema_pb.RecordValue
|
||||
|
recordValue, err := schema.ToRecordValue(recordType, parquetLevels, row) |
||||
|
if err != nil { |
||||
|
return processedTsNs, fmt.Errorf("ToRecordValue failed: %v", err) |
||||
|
} |
||||
|
processedTsNs = recordValue.Fields[SW_COLUMN_NAME_TS].GetInt64Value() |
||||
|
if processedTsNs < starTsNs { |
||||
|
continue |
||||
|
} |
||||
|
if stopTsNs != 0 && processedTsNs >= stopTsNs { |
||||
|
return processedTsNs, nil |
||||
|
} |
||||
|
|
||||
|
data, marshalErr := proto.Marshal(recordValue) |
||||
|
if marshalErr != nil { |
||||
|
return processedTsNs, fmt.Errorf("marshal record value: %v", marshalErr) |
||||
|
} |
||||
|
|
||||
|
logEntry := &filer_pb.LogEntry{ |
||||
|
Key: recordValue.Fields[SW_COLUMN_NAME_KEY].GetBytesValue(), |
||||
|
TsNs: processedTsNs, |
||||
|
Data: data, |
||||
|
} |
||||
|
|
||||
|
// fmt.Printf(" parquet entry %s ts %v\n", string(logEntry.Key), time.Unix(0, logEntry.TsNs).UTC())
|
||||
|
|
||||
|
if _, err = eachLogEntryFn(logEntry); err != nil { |
||||
|
return processedTsNs, fmt.Errorf("process log entry %v: %v", logEntry, err) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if readErr != nil { |
||||
|
if readErr == io.EOF { |
||||
|
return processedTsNs, nil |
||||
|
} |
||||
|
return processedTsNs, readErr |
||||
|
} |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
return func(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastReadPosition log_buffer.MessagePosition, isDone bool, err error) { |
||||
|
startFileName := startPosition.UTC().Format(topic.TIME_FORMAT) |
||||
|
startTsNs := startPosition.Time.UnixNano() |
||||
|
var processedTsNs int64 |
||||
|
|
||||
|
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
|
||||
|
return filer_pb.SeaweedList(client, partitionDir, "", func(entry *filer_pb.Entry, isLast bool) error { |
||||
|
if entry.IsDirectory { |
||||
|
return nil |
||||
|
} |
||||
|
if !strings.HasSuffix(entry.Name, ".parquet") { |
||||
|
return nil |
||||
|
} |
||||
|
if len(entry.Extended) == 0 { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// read minTs from the parquet file
|
||||
|
minTsBytes := entry.Extended["min"] |
||||
|
if len(minTsBytes) != 8 { |
||||
|
return nil |
||||
|
} |
||||
|
minTsNs := int64(binary.BigEndian.Uint64(minTsBytes)) |
||||
|
|
||||
|
// read max ts
|
||||
|
maxTsBytes := entry.Extended["max"] |
||||
|
if len(maxTsBytes) != 8 { |
||||
|
return nil |
||||
|
} |
||||
|
maxTsNs := int64(binary.BigEndian.Uint64(maxTsBytes)) |
||||
|
|
||||
|
if stopTsNs != 0 && stopTsNs <= minTsNs { |
||||
|
isDone = true |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
if maxTsNs < startTsNs { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
if processedTsNs, err = eachFileFn(entry, eachLogEntryFn, startTsNs, stopTsNs); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
return nil |
||||
|
|
||||
|
}, startFileName, true, math.MaxInt32) |
||||
|
}) |
||||
|
lastReadPosition = log_buffer.NewMessagePosition(processedTsNs, -2) |
||||
|
return |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,94 @@ |
|||||
|
package shell |
||||
|
|
||||
|
import ( |
||||
|
"flag" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/filer_client" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/logstore" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/schema" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/operation" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb" |
||||
|
"google.golang.org/grpc" |
||||
|
"io" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
Commands = append(Commands, &commandMqTopicCompact{}) |
||||
|
} |
||||
|
|
||||
|
type commandMqTopicCompact struct { |
||||
|
} |
||||
|
|
||||
|
func (c *commandMqTopicCompact) Name() string { |
||||
|
return "mq.topic.compact" |
||||
|
} |
||||
|
|
||||
|
func (c *commandMqTopicCompact) Help() string { |
||||
|
return `compact the topic storage into parquet format |
||||
|
|
||||
|
Example: |
||||
|
mq.topic.compact -namespace <namespace> -topic <topic_name> -timeAgo <time_ago> |
||||
|
|
||||
|
` |
||||
|
} |
||||
|
|
||||
|
func (c *commandMqTopicCompact) HasTag(tag CommandTag) bool { |
||||
|
return ResourceHeavy == tag |
||||
|
} |
||||
|
|
||||
|
func (c *commandMqTopicCompact) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error { |
||||
|
|
||||
|
// parse parameters
|
||||
|
mqCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) |
||||
|
namespace := mqCommand.String("namespace", "", "namespace name") |
||||
|
topicName := mqCommand.String("topic", "", "topic name") |
||||
|
timeAgo := mqCommand.Duration("timeAgo", 2*time.Minute, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") |
||||
|
replication := mqCommand.String("replication", "", "replication type") |
||||
|
collection := mqCommand.String("collection", "", "optional collection name") |
||||
|
dataCenter := mqCommand.String("dataCenter", "", "optional data center name") |
||||
|
diskType := mqCommand.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") |
||||
|
maxMB := mqCommand.Int("maxMB", 4, "split files larger than the limit") |
||||
|
|
||||
|
if err := mqCommand.Parse(args); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
storagePreference := &operation.StoragePreference{ |
||||
|
Replication: *replication, |
||||
|
Collection: *collection, |
||||
|
DataCenter: *dataCenter, |
||||
|
DiskType: *diskType, |
||||
|
MaxMB: *maxMB, |
||||
|
} |
||||
|
|
||||
|
// read topic configuration
|
||||
|
fca := &filer_client.FilerClientAccessor{ |
||||
|
GetFiler: func() pb.ServerAddress { |
||||
|
return commandEnv.option.FilerAddress |
||||
|
}, |
||||
|
GetGrpcDialOption: func() grpc.DialOption { |
||||
|
return commandEnv.option.GrpcDialOption |
||||
|
}, |
||||
|
} |
||||
|
t := topic.NewTopic(*namespace, *topicName) |
||||
|
topicConf, err := fca.ReadTopicConfFromFiler(t) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// get record type
|
||||
|
recordType := topicConf.GetRecordType() |
||||
|
recordType = schema.NewRecordTypeBuilder(recordType). |
||||
|
WithField(logstore.SW_COLUMN_NAME_TS, schema.TypeInt64). |
||||
|
WithField(logstore.SW_COLUMN_NAME_KEY, schema.TypeBytes). |
||||
|
RecordTypeEnd() |
||||
|
|
||||
|
// compact the topic partition versions
|
||||
|
if err = logstore.CompactTopicPartitions(commandEnv, t, *timeAgo, recordType, storagePreference); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
|
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue