diff --git a/weed/data/Makefile b/weed/data/Makefile new file mode 100644 index 000000000..bc6431a24 --- /dev/null +++ b/weed/data/Makefile @@ -0,0 +1,6 @@ +all: gen + +.PHONY : gen + +gen: + protoc columnar.proto --go_out=./columnar_pb --go-grpc_out=./columnar_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative diff --git a/weed/data/column_uint16.go b/weed/data/column_uint16.go new file mode 100644 index 000000000..3442df041 --- /dev/null +++ b/weed/data/column_uint16.go @@ -0,0 +1,32 @@ +package data + +import ( + "encoding/binary" + "fmt" + "io" +) + +type ColumnUint16 struct { +} + +const SIZE_Uint16 = 2 + +func (c *ColumnUint16) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint16 { + if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint16); n == SIZE_Uint16 && err == nil { + return binary.BigEndian.Uint16(buf) + } + return 0 +} + +func WriteUint16s(buf []byte, data []uint16) (err error) { + off := 0 + size := len(data) + if len(buf) < size<<1 { + return fmt.Errorf("buf too small") + } + for _, dat := range data { + binary.BigEndian.PutUint16(buf[off:], dat) + off += SIZE_Uint16 + } + return nil +} diff --git a/weed/data/column_uint32.go b/weed/data/column_uint32.go new file mode 100644 index 000000000..2b661786a --- /dev/null +++ b/weed/data/column_uint32.go @@ -0,0 +1,32 @@ +package data + +import ( + "encoding/binary" + "fmt" + "io" +) + +type ColumnUint32 struct { +} + +const SIZE_Uint32 = 4 + +func (c *ColumnUint32) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint32 { + if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint32); n == SIZE_Uint32 && err == nil { + return binary.BigEndian.Uint32(buf) + } + return 0 +} + +func WriteUint32s(buf []byte, data []uint32) (err error) { + off := 0 + size := len(data) + if len(buf) < size<<2 { + return fmt.Errorf("buf too small") + } + for _, dat := range data { + binary.BigEndian.PutUint32(buf[off:], dat) + off += SIZE_Uint32 + } + return nil +} diff --git a/weed/data/columnar.proto b/weed/data/columnar.proto new file mode 100644 index 000000000..e6ac692b5 --- /dev/null +++ b/weed/data/columnar.proto @@ -0,0 +1,103 @@ +syntax = "proto3"; + +package columnar_pb; + +option go_package = "github.com/seaweedfs/seaweedfs/weed/data/columnar_pb"; + +message FileId { + uint32 volume_id = 1; + uint64 file_key = 2; + fixed32 cookie = 3; +} + +enum LogicalType { + Uint8 = 0; + Uint16 = 1; + Float32 = 4; +} + +message ColumnUint16 { + uint32 base = 1; + uint32 min = 3; + uint32 max = 4; +} + +message ColumnUint32 { + uint32 base = 1; + uint32 min = 3; + uint32 max = 4; +} + +message ColumnFloat32 { + uint32 min = 3; + uint32 max = 4; +} + +message ColumnSplit { + // The ids of the fields/columns in this file + int32 field_id = 1; + FileId file_id = 2; + int64 row_offset = 3; + int32 row_count = 4; + + oneof storage_type { + ColumnUint16 meta_uint16 = 8; + ColumnUint32 meta_uint32 = 9; + ColumnFloat32 meta_float32 = 10; + } +} + +message Snapshot { + // All fields of the dataset, including the nested fields. + repeated Field fields = 1; + + repeated string data_files = 2; + + // Snapshot version number. + uint64 version = 3; + +} + +message DataFile { + repeated int32 field_ids = 1; + repeated RowGroup row_groups = 2; +} + +message RowGroup { + int64 row_offset = 1; + int32 row_count = 2; + repeated ColumnSplit column_splits = 3; +} + +// Field metadata for a column. +message Field { + enum Type { + PARENT = 0; + REPEATED = 1; + LEAF = 2; + } + Type type = 1; + + // Fully qualified name. + string name = 2; + /// Field Id. + int32 id = 3; + /// Parent Field ID. If not set, this is a top-level column. + int32 parent_id = 4; + + // Logical types, support parameterized Arrow Type. + LogicalType logical_type = 5; + // If this field is nullable. + bool nullable = 6; +} + + +message AnyValue { + oneof value { + bytes bytes_value = 1; + bool bool_value = 2; + uint64 int64_value = 3; + uint32 int32_value = 4; + double double_value = 5; + } +} diff --git a/weed/data/columnar_pb/columnar.pb.go b/weed/data/columnar_pb/columnar.pb.go new file mode 100644 index 000000000..3b1bde1cc --- /dev/null +++ b/weed/data/columnar_pb/columnar.pb.go @@ -0,0 +1,1199 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.11 +// source: columnar.proto + +package columnar_pb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LogicalType int32 + +const ( + LogicalType_Uint8 LogicalType = 0 + LogicalType_Uint16 LogicalType = 1 + LogicalType_Float32 LogicalType = 4 +) + +// Enum value maps for LogicalType. +var ( + LogicalType_name = map[int32]string{ + 0: "Uint8", + 1: "Uint16", + 4: "Float32", + } + LogicalType_value = map[string]int32{ + "Uint8": 0, + "Uint16": 1, + "Float32": 4, + } +) + +func (x LogicalType) Enum() *LogicalType { + p := new(LogicalType) + *p = x + return p +} + +func (x LogicalType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (LogicalType) Descriptor() protoreflect.EnumDescriptor { + return file_columnar_proto_enumTypes[0].Descriptor() +} + +func (LogicalType) Type() protoreflect.EnumType { + return &file_columnar_proto_enumTypes[0] +} + +func (x LogicalType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use LogicalType.Descriptor instead. +func (LogicalType) EnumDescriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{0} +} + +type Field_Type int32 + +const ( + Field_PARENT Field_Type = 0 + Field_REPEATED Field_Type = 1 + Field_LEAF Field_Type = 2 +) + +// Enum value maps for Field_Type. +var ( + Field_Type_name = map[int32]string{ + 0: "PARENT", + 1: "REPEATED", + 2: "LEAF", + } + Field_Type_value = map[string]int32{ + "PARENT": 0, + "REPEATED": 1, + "LEAF": 2, + } +) + +func (x Field_Type) Enum() *Field_Type { + p := new(Field_Type) + *p = x + return p +} + +func (x Field_Type) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Field_Type) Descriptor() protoreflect.EnumDescriptor { + return file_columnar_proto_enumTypes[1].Descriptor() +} + +func (Field_Type) Type() protoreflect.EnumType { + return &file_columnar_proto_enumTypes[1] +} + +func (x Field_Type) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Field_Type.Descriptor instead. +func (Field_Type) EnumDescriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{8, 0} +} + +type FileId struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + FileKey uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey,proto3" json:"file_key,omitempty"` + Cookie uint32 `protobuf:"fixed32,3,opt,name=cookie,proto3" json:"cookie,omitempty"` +} + +func (x *FileId) Reset() { + *x = FileId{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileId) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileId) ProtoMessage() {} + +func (x *FileId) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileId.ProtoReflect.Descriptor instead. +func (*FileId) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{0} +} + +func (x *FileId) GetVolumeId() uint32 { + if x != nil { + return x.VolumeId + } + return 0 +} + +func (x *FileId) GetFileKey() uint64 { + if x != nil { + return x.FileKey + } + return 0 +} + +func (x *FileId) GetCookie() uint32 { + if x != nil { + return x.Cookie + } + return 0 +} + +type ColumnUint16 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Base uint32 `protobuf:"varint,1,opt,name=base,proto3" json:"base,omitempty"` + Min uint32 `protobuf:"varint,3,opt,name=min,proto3" json:"min,omitempty"` + Max uint32 `protobuf:"varint,4,opt,name=max,proto3" json:"max,omitempty"` +} + +func (x *ColumnUint16) Reset() { + *x = ColumnUint16{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ColumnUint16) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ColumnUint16) ProtoMessage() {} + +func (x *ColumnUint16) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ColumnUint16.ProtoReflect.Descriptor instead. +func (*ColumnUint16) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{1} +} + +func (x *ColumnUint16) GetBase() uint32 { + if x != nil { + return x.Base + } + return 0 +} + +func (x *ColumnUint16) GetMin() uint32 { + if x != nil { + return x.Min + } + return 0 +} + +func (x *ColumnUint16) GetMax() uint32 { + if x != nil { + return x.Max + } + return 0 +} + +type ColumnUint32 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Base uint32 `protobuf:"varint,1,opt,name=base,proto3" json:"base,omitempty"` + Min uint32 `protobuf:"varint,3,opt,name=min,proto3" json:"min,omitempty"` + Max uint32 `protobuf:"varint,4,opt,name=max,proto3" json:"max,omitempty"` +} + +func (x *ColumnUint32) Reset() { + *x = ColumnUint32{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ColumnUint32) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ColumnUint32) ProtoMessage() {} + +func (x *ColumnUint32) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ColumnUint32.ProtoReflect.Descriptor instead. +func (*ColumnUint32) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{2} +} + +func (x *ColumnUint32) GetBase() uint32 { + if x != nil { + return x.Base + } + return 0 +} + +func (x *ColumnUint32) GetMin() uint32 { + if x != nil { + return x.Min + } + return 0 +} + +func (x *ColumnUint32) GetMax() uint32 { + if x != nil { + return x.Max + } + return 0 +} + +type ColumnFloat32 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Min uint32 `protobuf:"varint,3,opt,name=min,proto3" json:"min,omitempty"` + Max uint32 `protobuf:"varint,4,opt,name=max,proto3" json:"max,omitempty"` +} + +func (x *ColumnFloat32) Reset() { + *x = ColumnFloat32{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ColumnFloat32) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ColumnFloat32) ProtoMessage() {} + +func (x *ColumnFloat32) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ColumnFloat32.ProtoReflect.Descriptor instead. +func (*ColumnFloat32) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{3} +} + +func (x *ColumnFloat32) GetMin() uint32 { + if x != nil { + return x.Min + } + return 0 +} + +func (x *ColumnFloat32) GetMax() uint32 { + if x != nil { + return x.Max + } + return 0 +} + +type ColumnSplit struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The ids of the fields/columns in this file + FieldId int32 `protobuf:"varint,1,opt,name=field_id,json=fieldId,proto3" json:"field_id,omitempty"` + FileId *FileId `protobuf:"bytes,2,opt,name=file_id,json=fileId,proto3" json:"file_id,omitempty"` + RowOffset int64 `protobuf:"varint,3,opt,name=row_offset,json=rowOffset,proto3" json:"row_offset,omitempty"` + RowCount int32 `protobuf:"varint,4,opt,name=row_count,json=rowCount,proto3" json:"row_count,omitempty"` + // Types that are assignable to StorageType: + // + // *ColumnSplit_MetaUint16 + // *ColumnSplit_MetaUint32 + // *ColumnSplit_MetaFloat32 + StorageType isColumnSplit_StorageType `protobuf_oneof:"storage_type"` +} + +func (x *ColumnSplit) Reset() { + *x = ColumnSplit{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ColumnSplit) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ColumnSplit) ProtoMessage() {} + +func (x *ColumnSplit) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ColumnSplit.ProtoReflect.Descriptor instead. +func (*ColumnSplit) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{4} +} + +func (x *ColumnSplit) GetFieldId() int32 { + if x != nil { + return x.FieldId + } + return 0 +} + +func (x *ColumnSplit) GetFileId() *FileId { + if x != nil { + return x.FileId + } + return nil +} + +func (x *ColumnSplit) GetRowOffset() int64 { + if x != nil { + return x.RowOffset + } + return 0 +} + +func (x *ColumnSplit) GetRowCount() int32 { + if x != nil { + return x.RowCount + } + return 0 +} + +func (m *ColumnSplit) GetStorageType() isColumnSplit_StorageType { + if m != nil { + return m.StorageType + } + return nil +} + +func (x *ColumnSplit) GetMetaUint16() *ColumnUint16 { + if x, ok := x.GetStorageType().(*ColumnSplit_MetaUint16); ok { + return x.MetaUint16 + } + return nil +} + +func (x *ColumnSplit) GetMetaUint32() *ColumnUint32 { + if x, ok := x.GetStorageType().(*ColumnSplit_MetaUint32); ok { + return x.MetaUint32 + } + return nil +} + +func (x *ColumnSplit) GetMetaFloat32() *ColumnFloat32 { + if x, ok := x.GetStorageType().(*ColumnSplit_MetaFloat32); ok { + return x.MetaFloat32 + } + return nil +} + +type isColumnSplit_StorageType interface { + isColumnSplit_StorageType() +} + +type ColumnSplit_MetaUint16 struct { + MetaUint16 *ColumnUint16 `protobuf:"bytes,8,opt,name=meta_uint16,json=metaUint16,proto3,oneof"` +} + +type ColumnSplit_MetaUint32 struct { + MetaUint32 *ColumnUint32 `protobuf:"bytes,9,opt,name=meta_uint32,json=metaUint32,proto3,oneof"` +} + +type ColumnSplit_MetaFloat32 struct { + MetaFloat32 *ColumnFloat32 `protobuf:"bytes,10,opt,name=meta_float32,json=metaFloat32,proto3,oneof"` +} + +func (*ColumnSplit_MetaUint16) isColumnSplit_StorageType() {} + +func (*ColumnSplit_MetaUint32) isColumnSplit_StorageType() {} + +func (*ColumnSplit_MetaFloat32) isColumnSplit_StorageType() {} + +type Snapshot struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // All fields of the dataset, including the nested fields. + Fields []*Field `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty"` + DataFiles []string `protobuf:"bytes,2,rep,name=data_files,json=dataFiles,proto3" json:"data_files,omitempty"` + // Snapshot version number. + Version uint64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` +} + +func (x *Snapshot) Reset() { + *x = Snapshot{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Snapshot) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Snapshot) ProtoMessage() {} + +func (x *Snapshot) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Snapshot.ProtoReflect.Descriptor instead. +func (*Snapshot) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{5} +} + +func (x *Snapshot) GetFields() []*Field { + if x != nil { + return x.Fields + } + return nil +} + +func (x *Snapshot) GetDataFiles() []string { + if x != nil { + return x.DataFiles + } + return nil +} + +func (x *Snapshot) GetVersion() uint64 { + if x != nil { + return x.Version + } + return 0 +} + +type DataFile struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FieldIds []int32 `protobuf:"varint,1,rep,packed,name=field_ids,json=fieldIds,proto3" json:"field_ids,omitempty"` + RowGroups []*RowGroup `protobuf:"bytes,2,rep,name=row_groups,json=rowGroups,proto3" json:"row_groups,omitempty"` +} + +func (x *DataFile) Reset() { + *x = DataFile{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DataFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DataFile) ProtoMessage() {} + +func (x *DataFile) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DataFile.ProtoReflect.Descriptor instead. +func (*DataFile) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{6} +} + +func (x *DataFile) GetFieldIds() []int32 { + if x != nil { + return x.FieldIds + } + return nil +} + +func (x *DataFile) GetRowGroups() []*RowGroup { + if x != nil { + return x.RowGroups + } + return nil +} + +type RowGroup struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + RowOffset int64 `protobuf:"varint,1,opt,name=row_offset,json=rowOffset,proto3" json:"row_offset,omitempty"` + RowCount int32 `protobuf:"varint,2,opt,name=row_count,json=rowCount,proto3" json:"row_count,omitempty"` + ColumnSplits []*ColumnSplit `protobuf:"bytes,3,rep,name=column_splits,json=columnSplits,proto3" json:"column_splits,omitempty"` +} + +func (x *RowGroup) Reset() { + *x = RowGroup{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RowGroup) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RowGroup) ProtoMessage() {} + +func (x *RowGroup) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RowGroup.ProtoReflect.Descriptor instead. +func (*RowGroup) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{7} +} + +func (x *RowGroup) GetRowOffset() int64 { + if x != nil { + return x.RowOffset + } + return 0 +} + +func (x *RowGroup) GetRowCount() int32 { + if x != nil { + return x.RowCount + } + return 0 +} + +func (x *RowGroup) GetColumnSplits() []*ColumnSplit { + if x != nil { + return x.ColumnSplits + } + return nil +} + +// Field metadata for a column. +type Field struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type Field_Type `protobuf:"varint,1,opt,name=type,proto3,enum=columnar_pb.Field_Type" json:"type,omitempty"` + // Fully qualified name. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + // / Field Id. + Id int32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"` + // / Parent Field ID. If not set, this is a top-level column. + ParentId int32 `protobuf:"varint,4,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` + // Logical types, support parameterized Arrow Type. + LogicalType LogicalType `protobuf:"varint,5,opt,name=logical_type,json=logicalType,proto3,enum=columnar_pb.LogicalType" json:"logical_type,omitempty"` + // If this field is nullable. + Nullable bool `protobuf:"varint,6,opt,name=nullable,proto3" json:"nullable,omitempty"` +} + +func (x *Field) Reset() { + *x = Field{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Field) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Field) ProtoMessage() {} + +func (x *Field) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Field.ProtoReflect.Descriptor instead. +func (*Field) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{8} +} + +func (x *Field) GetType() Field_Type { + if x != nil { + return x.Type + } + return Field_PARENT +} + +func (x *Field) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Field) GetId() int32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *Field) GetParentId() int32 { + if x != nil { + return x.ParentId + } + return 0 +} + +func (x *Field) GetLogicalType() LogicalType { + if x != nil { + return x.LogicalType + } + return LogicalType_Uint8 +} + +func (x *Field) GetNullable() bool { + if x != nil { + return x.Nullable + } + return false +} + +type AnyValue struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Value: + // + // *AnyValue_BytesValue + // *AnyValue_BoolValue + // *AnyValue_Int64Value + // *AnyValue_Int32Value + // *AnyValue_DoubleValue + Value isAnyValue_Value `protobuf_oneof:"value"` +} + +func (x *AnyValue) Reset() { + *x = AnyValue{} + if protoimpl.UnsafeEnabled { + mi := &file_columnar_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AnyValue) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AnyValue) ProtoMessage() {} + +func (x *AnyValue) ProtoReflect() protoreflect.Message { + mi := &file_columnar_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AnyValue.ProtoReflect.Descriptor instead. +func (*AnyValue) Descriptor() ([]byte, []int) { + return file_columnar_proto_rawDescGZIP(), []int{9} +} + +func (m *AnyValue) GetValue() isAnyValue_Value { + if m != nil { + return m.Value + } + return nil +} + +func (x *AnyValue) GetBytesValue() []byte { + if x, ok := x.GetValue().(*AnyValue_BytesValue); ok { + return x.BytesValue + } + return nil +} + +func (x *AnyValue) GetBoolValue() bool { + if x, ok := x.GetValue().(*AnyValue_BoolValue); ok { + return x.BoolValue + } + return false +} + +func (x *AnyValue) GetInt64Value() uint64 { + if x, ok := x.GetValue().(*AnyValue_Int64Value); ok { + return x.Int64Value + } + return 0 +} + +func (x *AnyValue) GetInt32Value() uint32 { + if x, ok := x.GetValue().(*AnyValue_Int32Value); ok { + return x.Int32Value + } + return 0 +} + +func (x *AnyValue) GetDoubleValue() float64 { + if x, ok := x.GetValue().(*AnyValue_DoubleValue); ok { + return x.DoubleValue + } + return 0 +} + +type isAnyValue_Value interface { + isAnyValue_Value() +} + +type AnyValue_BytesValue struct { + BytesValue []byte `protobuf:"bytes,1,opt,name=bytes_value,json=bytesValue,proto3,oneof"` +} + +type AnyValue_BoolValue struct { + BoolValue bool `protobuf:"varint,2,opt,name=bool_value,json=boolValue,proto3,oneof"` +} + +type AnyValue_Int64Value struct { + Int64Value uint64 `protobuf:"varint,3,opt,name=int64_value,json=int64Value,proto3,oneof"` +} + +type AnyValue_Int32Value struct { + Int32Value uint32 `protobuf:"varint,4,opt,name=int32_value,json=int32Value,proto3,oneof"` +} + +type AnyValue_DoubleValue struct { + DoubleValue float64 `protobuf:"fixed64,5,opt,name=double_value,json=doubleValue,proto3,oneof"` +} + +func (*AnyValue_BytesValue) isAnyValue_Value() {} + +func (*AnyValue_BoolValue) isAnyValue_Value() {} + +func (*AnyValue_Int64Value) isAnyValue_Value() {} + +func (*AnyValue_Int32Value) isAnyValue_Value() {} + +func (*AnyValue_DoubleValue) isAnyValue_Value() {} + +var File_columnar_proto protoreflect.FileDescriptor + +var file_columnar_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x22, 0x58, 0x0a, + 0x06, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x07, 0x52, + 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x22, 0x46, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x55, 0x69, 0x6e, 0x74, 0x31, 0x36, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x62, 0x61, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, + 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, + 0x03, 0x6d, 0x61, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, + 0x46, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x55, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x12, + 0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x62, + 0x61, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, 0x33, 0x0a, 0x0d, 0x43, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, + 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, 0xdf, 0x02, 0x0a, + 0x0b, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x52, 0x06, 0x66, + 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x6f, 0x77, 0x5f, 0x6f, 0x66, 0x66, + 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x72, 0x6f, 0x77, 0x4f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x6f, 0x77, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x12, 0x3c, 0x0a, 0x0b, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x75, 0x69, 0x6e, 0x74, 0x31, 0x36, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x55, 0x69, 0x6e, 0x74, 0x31, + 0x36, 0x48, 0x00, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x61, 0x55, 0x69, 0x6e, 0x74, 0x31, 0x36, 0x12, + 0x3c, 0x0a, 0x0b, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x75, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x55, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x48, + 0x00, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x61, 0x55, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x12, 0x3f, 0x0a, + 0x0c, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x48, + 0x00, 0x52, 0x0b, 0x6d, 0x65, 0x74, 0x61, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x42, 0x0e, + 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x6f, + 0x0a, 0x08, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x2a, 0x0a, 0x06, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, + 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61, + 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, + 0x5d, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x64, 0x73, 0x12, 0x34, 0x0a, 0x0a, 0x72, 0x6f, 0x77, 0x5f, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x77, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x52, 0x09, 0x72, 0x6f, 0x77, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0x85, + 0x01, 0x0a, 0x08, 0x52, 0x6f, 0x77, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x72, + 0x6f, 0x77, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x09, 0x72, 0x6f, 0x77, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f, + 0x77, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, + 0x6f, 0x77, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x3d, 0x0a, 0x0d, 0x63, 0x6f, 0x6c, 0x75, 0x6d, + 0x6e, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, + 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, + 0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x22, 0xfa, 0x01, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, + 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x3b, + 0x0a, 0x0c, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, + 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6e, + 0x75, 0x6c, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6e, + 0x75, 0x6c, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x2a, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x0a, 0x0a, 0x06, 0x50, 0x41, 0x52, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, + 0x45, 0x50, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x45, 0x41, + 0x46, 0x10, 0x02, 0x22, 0xc2, 0x01, 0x0a, 0x08, 0x41, 0x6e, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x12, 0x21, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x5f, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x0a, 0x69, 0x6e, 0x74, + 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x33, 0x32, + 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x0a, + 0x69, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, + 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, + 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, + 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2a, 0x31, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x69, + 0x63, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x69, 0x6e, 0x74, 0x38, + 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x69, 0x6e, 0x74, 0x31, 0x36, 0x10, 0x01, 0x12, 0x0b, + 0x0a, 0x07, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x10, 0x04, 0x42, 0x36, 0x5a, 0x34, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, + 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, + 0x65, 0x64, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, + 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_columnar_proto_rawDescOnce sync.Once + file_columnar_proto_rawDescData = file_columnar_proto_rawDesc +) + +func file_columnar_proto_rawDescGZIP() []byte { + file_columnar_proto_rawDescOnce.Do(func() { + file_columnar_proto_rawDescData = protoimpl.X.CompressGZIP(file_columnar_proto_rawDescData) + }) + return file_columnar_proto_rawDescData +} + +var file_columnar_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_columnar_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_columnar_proto_goTypes = []interface{}{ + (LogicalType)(0), // 0: columnar_pb.LogicalType + (Field_Type)(0), // 1: columnar_pb.Field.Type + (*FileId)(nil), // 2: columnar_pb.FileId + (*ColumnUint16)(nil), // 3: columnar_pb.ColumnUint16 + (*ColumnUint32)(nil), // 4: columnar_pb.ColumnUint32 + (*ColumnFloat32)(nil), // 5: columnar_pb.ColumnFloat32 + (*ColumnSplit)(nil), // 6: columnar_pb.ColumnSplit + (*Snapshot)(nil), // 7: columnar_pb.Snapshot + (*DataFile)(nil), // 8: columnar_pb.DataFile + (*RowGroup)(nil), // 9: columnar_pb.RowGroup + (*Field)(nil), // 10: columnar_pb.Field + (*AnyValue)(nil), // 11: columnar_pb.AnyValue +} +var file_columnar_proto_depIdxs = []int32{ + 2, // 0: columnar_pb.ColumnSplit.file_id:type_name -> columnar_pb.FileId + 3, // 1: columnar_pb.ColumnSplit.meta_uint16:type_name -> columnar_pb.ColumnUint16 + 4, // 2: columnar_pb.ColumnSplit.meta_uint32:type_name -> columnar_pb.ColumnUint32 + 5, // 3: columnar_pb.ColumnSplit.meta_float32:type_name -> columnar_pb.ColumnFloat32 + 10, // 4: columnar_pb.Snapshot.fields:type_name -> columnar_pb.Field + 9, // 5: columnar_pb.DataFile.row_groups:type_name -> columnar_pb.RowGroup + 6, // 6: columnar_pb.RowGroup.column_splits:type_name -> columnar_pb.ColumnSplit + 1, // 7: columnar_pb.Field.type:type_name -> columnar_pb.Field.Type + 0, // 8: columnar_pb.Field.logical_type:type_name -> columnar_pb.LogicalType + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name +} + +func init() { file_columnar_proto_init() } +func file_columnar_proto_init() { + if File_columnar_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_columnar_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileId); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ColumnUint16); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ColumnUint32); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ColumnFloat32); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ColumnSplit); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Snapshot); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataFile); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RowGroup); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Field); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_columnar_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AnyValue); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_columnar_proto_msgTypes[4].OneofWrappers = []interface{}{ + (*ColumnSplit_MetaUint16)(nil), + (*ColumnSplit_MetaUint32)(nil), + (*ColumnSplit_MetaFloat32)(nil), + } + file_columnar_proto_msgTypes[9].OneofWrappers = []interface{}{ + (*AnyValue_BytesValue)(nil), + (*AnyValue_BoolValue)(nil), + (*AnyValue_Int64Value)(nil), + (*AnyValue_Int32Value)(nil), + (*AnyValue_DoubleValue)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_columnar_proto_rawDesc, + NumEnums: 2, + NumMessages: 10, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_columnar_proto_goTypes, + DependencyIndexes: file_columnar_proto_depIdxs, + EnumInfos: file_columnar_proto_enumTypes, + MessageInfos: file_columnar_proto_msgTypes, + }.Build() + File_columnar_proto = out.File + file_columnar_proto_rawDesc = nil + file_columnar_proto_goTypes = nil + file_columnar_proto_depIdxs = nil +} diff --git a/weed/data/datum.go b/weed/data/datum.go new file mode 100644 index 000000000..5805b691d --- /dev/null +++ b/weed/data/datum.go @@ -0,0 +1,69 @@ +package data + +import "fmt" + +type Datum interface { + Compare(other Datum) (int, error) +} +type Datums []Datum + +type DUint16 uint16 +type DUint32 uint32 +type dNull struct{} + +var ( + DNull Datum = dNull{} +) + +func (d dNull) Compare(other Datum) (int, error) { + if other == DNull { + return 0, nil + } + return -1, nil +} + +func NewDUint16(d DUint16) *DUint16 { + return &d +} +func NewDUint32(d DUint32) *DUint32 { + return &d +} + +func (d *DUint16) Compare(other Datum) (int, error) { + if other == DNull { + return 1, nil + } + thisV := *d + var otherV DUint16 + switch t := other.(type) { + case *DUint16: + otherV = *t + default: + return 0, fmt.Errorf("unsupported") + } + if thisV < otherV { + return -1, nil + } + if thisV > otherV { + return 1, nil + } + return 0, nil +} +func (d *DUint32) Compare(other Datum) (int, error) { + if other == DNull { + return 1, nil + } + thisV := *d + var otherV DUint32 + switch t := other.(type) { + case *DUint32: + otherV = *t + } + if thisV < otherV { + return -1, nil + } + if thisV > otherV { + return 1, nil + } + return 0, nil +} diff --git a/weed/data/read_test.go b/weed/data/read_test.go new file mode 100644 index 000000000..83742d675 --- /dev/null +++ b/weed/data/read_test.go @@ -0,0 +1,194 @@ +package data + +import ( + "encoding/binary" + "fmt" + "github.com/seaweedfs/seaweedfs/weed/util" + "io" + "testing" +) + +func TestRead(t *testing.T) { + x := make([]uint16, 128) + y := make([]uint32, 128) + + for i := range x { + x[i] = uint16(i) + } + for i := range y { + y[i] = uint32(i * 32) + } + + xbuf := make([]byte, len(x)*SIZE_Uint16) + ybuf := make([]byte, len(x)*SIZE_Uint32) + + WriteUint16s(xbuf, x) + WriteUint32s(ybuf, y) + + df := &DataFile{ + xbuf: xbuf, + ybuf: ybuf, + xLen: len(xbuf), + yLen: len(ybuf), + xReaderAt: util.NewBytesReader(xbuf), + yReaderAt: util.NewBytesReader(ybuf), + } + + dataLayout := make(map[FieldName]DataLayout) + dataLayout["x"] = DataLayout{ + LayoutType: Uint16, + SortType: Unsorted, + } + dataLayout["y"] = DataLayout{ + LayoutType: Uint32, + SortType: Unsorted, + } + + rows, err := df.ReadRows("x", dataLayout, Equal, NewDUint16(65)) + if err != nil { + fmt.Printf("err: %v", err) + return + } + for _, row := range rows { + fmt.Printf("row %d width %d ", row.index, len(row.Datums)) + for i, d := range row.Datums { + fmt.Printf("%d: %v ", i, d) + } + fmt.Println() + } + +} + +type Operator int32 +type LayoutType int32 +type SortType int32 + +const ( + Equal Operator = 0 + GreaterThan + GreaterOrEqual + LessThan + LessOrEqual + + Uint16 LayoutType = 0 + Uint32 = 1 + + Unsorted SortType = 0 + Ascending + Descending +) + +type DataFile struct { + xbuf []byte + ybuf []byte + xReaderAt io.ReaderAt + xLen int + yReaderAt io.ReaderAt + yLen int +} + +type DataLayout struct { + LayoutType + SortType +} + +type FieldName string + +func (d *DataFile) ReadRows(field FieldName, layout map[FieldName]DataLayout, op Operator, operand Datum) (rows []*Row, err error) { + if field == "x" { + rows, err = pushDownReadRows(d.xReaderAt, d.xLen, layout[field], op, operand) + if err != nil { + return + } + err = hydrateRows(d.yReaderAt, d.yLen, layout["y"], rows) + } + if field == "y" { + rows, err = pushDownReadRows(d.yReaderAt, d.yLen, layout[field], op, operand) + if err != nil { + return + } + err = hydrateRows(d.xReaderAt, d.xLen, layout["x"], rows) + } + return +} + +type Row struct { + index int + Datums +} + +func pushDownReadRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, op Operator, operand Datum) (rows []*Row, err error) { + if layout.LayoutType == Uint16 { + if layout.SortType == Unsorted { + buf := make([]byte, SIZE_Uint16) + for i := 0; i < dataLen; i += SIZE_Uint16 { + if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint16 && err == nil { + d := NewDUint16(DUint16(binary.BigEndian.Uint16(buf))) + cmp, err := d.Compare(operand) + if err != nil { + return rows, err + } + if cmp == 0 && op == Equal { + println(1) + rows = append(rows, &Row{ + index: i / SIZE_Uint16, + Datums: []Datum{d}, + }) + } + } + } + } + } + if layout.LayoutType == Uint32 { + if layout.SortType == Unsorted { + buf := make([]byte, SIZE_Uint32) + for i := 0; i < dataLen; i += SIZE_Uint32 { + if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint32 && err == nil { + d := NewDUint32(DUint32(binary.BigEndian.Uint32(buf))) + cmp, err := d.Compare(operand) + if err != nil { + return rows, err + } + if cmp == 0 && op == Equal { + println(2) + rows = append(rows, &Row{ + index: i / SIZE_Uint32, + Datums: []Datum{d}, + }) + } + } + } + } + } + return +} + +func hydrateRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, rows []*Row) (err error) { + if layout.LayoutType == Uint16 { + if layout.SortType == Unsorted { + buf := make([]byte, SIZE_Uint16) + for _, row := range rows { + if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint16); n == SIZE_Uint16 && err == nil { + t := binary.BigEndian.Uint16(buf) + d := NewDUint16(DUint16(t)) + println(3, "add", t) + row.Datums = append(row.Datums, d) + } + } + } + } + if layout.LayoutType == Uint32 { + if layout.SortType == Unsorted { + buf := make([]byte, SIZE_Uint32) + for _, row := range rows { + if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint32); n == SIZE_Uint32 && err == nil { + t := binary.BigEndian.Uint32(buf) + d := NewDUint32(DUint32(t)) + println(4, "add", t) + row.Datums = append(row.Datums, d) + } + } + } + } + return +}