Browse Source

parquet file generate stats

pull/7185/head
chrislu 1 month ago
parent
commit
ada704ad80
  1. 6
      weed/mq/logstore/log_to_parquet.go

6
weed/mq/logstore/log_to_parquet.go

@ -218,7 +218,11 @@ func writeLogFilesToParquet(filerClient filer_pb.FilerClient, partitionDir strin
os.Remove(tempFile.Name()) os.Remove(tempFile.Name())
}() }()
writer := parquet.NewWriter(tempFile, parquetSchema, parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel}))
// Enable column statistics for fast aggregation queries
writer := parquet.NewWriter(tempFile, parquetSchema,
parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel}),
parquet.DataPageStatistics(true), // Enable column statistics
)
rowBuilder := parquet.NewRowBuilder(parquetSchema) rowBuilder := parquet.NewRowBuilder(parquetSchema)
var startTsNs, stopTsNs int64 var startTsNs, stopTsNs int64

Loading…
Cancel
Save