joeslay
5 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1631 additions and 157 deletions
-
2README.md
-
5go.sum
-
2other/java/client/pom.xml
-
23other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
-
12other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
-
4other/java/client/src/test/java/seaweedfs/client/SeaweedFilerTest.java
-
6other/java/hdfs2/dependency-reduced-pom.xml
-
6other/java/hdfs2/pom.xml
-
2other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
6other/java/hdfs3/dependency-reduced-pom.xml
-
6other/java/hdfs3/pom.xml
-
2other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
61weed/pb/volume_server.proto
-
696weed/pb/volume_server_pb/volume_server.pb.go
-
107weed/query/json/query_json.go
-
133weed/query/json/query_json_test.go
-
17weed/query/json/seralize.go
-
124weed/query/sqltypes/const.go
-
101weed/query/sqltypes/type.go
-
30weed/query/sqltypes/unsafe.go
-
355weed/query/sqltypes/value.go
-
4weed/server/volume_grpc_copy.go
-
69weed/server/volume_grpc_query.go
-
3weed/storage/volume_loading.go
-
4weed/storage/volume_super_block.go
-
2weed/util/constants.go
-
6weed/wdclient/masterclient.go
@ -0,0 +1,107 @@ |
|||
package json |
|||
|
|||
import ( |
|||
"strconv" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/query/sqltypes" |
|||
"github.com/tidwall/gjson" |
|||
"github.com/tidwall/match" |
|||
) |
|||
|
|||
type Query struct { |
|||
Field string |
|||
Op string |
|||
Value string |
|||
} |
|||
|
|||
func QueryJson(jsonLine string, projections []string, query Query) (passedFilter bool, values []sqltypes.Value) { |
|||
if filterJson(jsonLine, query) { |
|||
passedFilter = true |
|||
fields := gjson.GetMany(jsonLine, projections...) |
|||
for _, f := range fields { |
|||
values = append(values, sqltypes.MakeTrusted(sqltypes.Type(f.Type), sqltypes.StringToBytes(f.Raw))) |
|||
} |
|||
return |
|||
} |
|||
return false, nil |
|||
} |
|||
|
|||
func filterJson(jsonLine string, query Query) bool { |
|||
|
|||
value := gjson.Get(jsonLine, query.Field) |
|||
|
|||
// copied from gjson.go queryMatches() function
|
|||
rpv := query.Value |
|||
|
|||
if !value.Exists() { |
|||
return false |
|||
} |
|||
if query.Op == "" { |
|||
// the query is only looking for existence, such as:
|
|||
// friends.#(name)
|
|||
// which makes sure that the array "friends" has an element of
|
|||
// "name" that exists
|
|||
return true |
|||
} |
|||
switch value.Type { |
|||
case gjson.String: |
|||
switch query.Op { |
|||
case "=": |
|||
return value.Str == rpv |
|||
case "!=": |
|||
return value.Str != rpv |
|||
case "<": |
|||
return value.Str < rpv |
|||
case "<=": |
|||
return value.Str <= rpv |
|||
case ">": |
|||
return value.Str > rpv |
|||
case ">=": |
|||
return value.Str >= rpv |
|||
case "%": |
|||
return match.Match(value.Str, rpv) |
|||
case "!%": |
|||
return !match.Match(value.Str, rpv) |
|||
} |
|||
case gjson.Number: |
|||
rpvn, _ := strconv.ParseFloat(rpv, 64) |
|||
switch query.Op { |
|||
case "=": |
|||
return value.Num == rpvn |
|||
case "!=": |
|||
return value.Num != rpvn |
|||
case "<": |
|||
return value.Num < rpvn |
|||
case "<=": |
|||
return value.Num <= rpvn |
|||
case ">": |
|||
return value.Num > rpvn |
|||
case ">=": |
|||
return value.Num >= rpvn |
|||
} |
|||
case gjson.True: |
|||
switch query.Op { |
|||
case "=": |
|||
return rpv == "true" |
|||
case "!=": |
|||
return rpv != "true" |
|||
case ">": |
|||
return rpv == "false" |
|||
case ">=": |
|||
return true |
|||
} |
|||
case gjson.False: |
|||
switch query.Op { |
|||
case "=": |
|||
return rpv == "false" |
|||
case "!=": |
|||
return rpv != "false" |
|||
case "<": |
|||
return rpv == "true" |
|||
case "<=": |
|||
return true |
|||
} |
|||
} |
|||
return false |
|||
|
|||
} |
|||
@ -0,0 +1,133 @@ |
|||
package json |
|||
|
|||
import ( |
|||
"testing" |
|||
|
|||
"github.com/tidwall/gjson" |
|||
) |
|||
|
|||
func TestGjson(t *testing.T) { |
|||
data := ` |
|||
{ |
|||
"quiz": { |
|||
"sport": { |
|||
"q1": { |
|||
"question": "Which one is correct team name in NBA?", |
|||
"options": [ |
|||
"New York Bulls", |
|||
"Los Angeles Kings", |
|||
"Golden State Warriros", |
|||
"Huston Rocket" |
|||
], |
|||
"answer": "Huston Rocket" |
|||
} |
|||
}, |
|||
"maths": { |
|||
"q1": { |
|||
"question": "5 + 7 = ?", |
|||
"options": [ |
|||
"10", |
|||
"11", |
|||
"12", |
|||
"13" |
|||
], |
|||
"answer": "12" |
|||
}, |
|||
"q2": { |
|||
"question": "12 - 8 = ?", |
|||
"options": [ |
|||
"1", |
|||
"2", |
|||
"3", |
|||
"4" |
|||
], |
|||
"answer": "4" |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
{ |
|||
"fruit": "Apple", |
|||
"size": "Large", |
|||
"quiz": "Red" |
|||
} |
|||
|
|||
` |
|||
|
|||
projections := []string{"quiz", "fruit"} |
|||
|
|||
gjson.ForEachLine(data, func(line gjson.Result) bool { |
|||
println(line.Raw) |
|||
println("+++++++++++") |
|||
results := gjson.GetMany(line.Raw, projections...) |
|||
for _, result := range results { |
|||
println(result.Index, result.Type, result.String()) |
|||
} |
|||
println("-----------") |
|||
return true |
|||
}) |
|||
|
|||
} |
|||
|
|||
func TestJsonQueryRow(t *testing.T) { |
|||
|
|||
data := ` |
|||
{ |
|||
"fruit": "Bl\"ue", |
|||
"size": 6, |
|||
"quiz": "green" |
|||
} |
|||
|
|||
` |
|||
selections := []string{"fruit", "size"} |
|||
|
|||
isFiltered, values := QueryJson(data, selections, Query{ |
|||
Field: "quiz", |
|||
Op: "=", |
|||
Value: "green", |
|||
}) |
|||
|
|||
if !isFiltered { |
|||
t.Errorf("should have been filtered") |
|||
} |
|||
|
|||
if values == nil { |
|||
t.Errorf("values should have been returned") |
|||
} |
|||
|
|||
buf := ToJson(nil, selections, values) |
|||
println(string(buf)) |
|||
|
|||
} |
|||
|
|||
func TestJsonQueryNumber(t *testing.T) { |
|||
|
|||
data := ` |
|||
{ |
|||
"fruit": "Bl\"ue", |
|||
"size": 6, |
|||
"quiz": "green" |
|||
} |
|||
|
|||
` |
|||
selections := []string{"fruit", "quiz"} |
|||
|
|||
isFiltered, values := QueryJson(data, selections, Query{ |
|||
Field: "size", |
|||
Op: ">=", |
|||
Value: "6", |
|||
}) |
|||
|
|||
if !isFiltered { |
|||
t.Errorf("should have been filtered") |
|||
} |
|||
|
|||
if values == nil { |
|||
t.Errorf("values should have been returned") |
|||
} |
|||
|
|||
buf := ToJson(nil, selections, values) |
|||
println(string(buf)) |
|||
|
|||
} |
|||
@ -0,0 +1,17 @@ |
|||
package json |
|||
|
|||
import "github.com/chrislusf/seaweedfs/weed/query/sqltypes" |
|||
|
|||
func ToJson(buf []byte, selections []string, values []sqltypes.Value) []byte { |
|||
buf = append(buf, '{') |
|||
for i, value := range values { |
|||
if i > 0 { |
|||
buf = append(buf, ',') |
|||
} |
|||
buf = append(buf, selections[i]...) |
|||
buf = append(buf, ':') |
|||
buf = append(buf, value.Raw()...) |
|||
} |
|||
buf = append(buf, '}') |
|||
return buf |
|||
} |
|||
@ -0,0 +1,124 @@ |
|||
package sqltypes |
|||
|
|||
// copied from vitness
|
|||
|
|||
// Flag allows us to qualify types by their common properties.
|
|||
type Flag int32 |
|||
|
|||
const ( |
|||
Flag_NONE Flag = 0 |
|||
Flag_ISINTEGRAL Flag = 256 |
|||
Flag_ISUNSIGNED Flag = 512 |
|||
Flag_ISFLOAT Flag = 1024 |
|||
Flag_ISQUOTED Flag = 2048 |
|||
Flag_ISTEXT Flag = 4096 |
|||
Flag_ISBINARY Flag = 8192 |
|||
) |
|||
|
|||
var Flag_name = map[int32]string{ |
|||
0: "NONE", |
|||
256: "ISINTEGRAL", |
|||
512: "ISUNSIGNED", |
|||
1024: "ISFLOAT", |
|||
2048: "ISQUOTED", |
|||
4096: "ISTEXT", |
|||
8192: "ISBINARY", |
|||
} |
|||
var Flag_value = map[string]int32{ |
|||
"NONE": 0, |
|||
"ISINTEGRAL": 256, |
|||
"ISUNSIGNED": 512, |
|||
"ISFLOAT": 1024, |
|||
"ISQUOTED": 2048, |
|||
"ISTEXT": 4096, |
|||
"ISBINARY": 8192, |
|||
} |
|||
|
|||
// Type defines the various supported data types in bind vars
|
|||
// and query results.
|
|||
type Type int32 |
|||
|
|||
const ( |
|||
// NULL_TYPE specifies a NULL type.
|
|||
Type_NULL_TYPE Type = 0 |
|||
// INT8 specifies a TINYINT type.
|
|||
// Properties: 1, IsNumber.
|
|||
Type_INT8 Type = 257 |
|||
// UINT8 specifies a TINYINT UNSIGNED type.
|
|||
// Properties: 2, IsNumber, IsUnsigned.
|
|||
Type_UINT8 Type = 770 |
|||
// INT16 specifies a SMALLINT type.
|
|||
// Properties: 3, IsNumber.
|
|||
Type_INT16 Type = 259 |
|||
// UINT16 specifies a SMALLINT UNSIGNED type.
|
|||
// Properties: 4, IsNumber, IsUnsigned.
|
|||
Type_UINT16 Type = 772 |
|||
// INT24 specifies a MEDIUMINT type.
|
|||
// Properties: 5, IsNumber.
|
|||
Type_INT32 Type = 263 |
|||
// UINT32 specifies a INTEGER UNSIGNED type.
|
|||
// Properties: 8, IsNumber, IsUnsigned.
|
|||
Type_UINT32 Type = 776 |
|||
// INT64 specifies a BIGINT type.
|
|||
// Properties: 9, IsNumber.
|
|||
Type_INT64 Type = 265 |
|||
// UINT64 specifies a BIGINT UNSIGNED type.
|
|||
// Properties: 10, IsNumber, IsUnsigned.
|
|||
Type_UINT64 Type = 778 |
|||
// FLOAT32 specifies a FLOAT type.
|
|||
// Properties: 11, IsFloat.
|
|||
Type_FLOAT32 Type = 1035 |
|||
// FLOAT64 specifies a DOUBLE or REAL type.
|
|||
// Properties: 12, IsFloat.
|
|||
Type_FLOAT64 Type = 1036 |
|||
// TIMESTAMP specifies a TIMESTAMP type.
|
|||
// Properties: 13, IsQuoted.
|
|||
Type_TIMESTAMP Type = 2061 |
|||
// DATE specifies a DATE type.
|
|||
// Properties: 14, IsQuoted.
|
|||
Type_DATE Type = 2062 |
|||
// TIME specifies a TIME type.
|
|||
// Properties: 15, IsQuoted.
|
|||
Type_TIME Type = 2063 |
|||
// DATETIME specifies a DATETIME type.
|
|||
// Properties: 16, IsQuoted.
|
|||
Type_DATETIME Type = 2064 |
|||
// YEAR specifies a YEAR type.
|
|||
// Properties: 17, IsNumber, IsUnsigned.
|
|||
Type_YEAR Type = 785 |
|||
// DECIMAL specifies a DECIMAL or NUMERIC type.
|
|||
// Properties: 18, None.
|
|||
Type_DECIMAL Type = 18 |
|||
// TEXT specifies a TEXT type.
|
|||
// Properties: 19, IsQuoted, IsText.
|
|||
Type_TEXT Type = 6163 |
|||
// BLOB specifies a BLOB type.
|
|||
// Properties: 20, IsQuoted, IsBinary.
|
|||
Type_BLOB Type = 10260 |
|||
// VARCHAR specifies a VARCHAR type.
|
|||
// Properties: 21, IsQuoted, IsText.
|
|||
Type_VARCHAR Type = 6165 |
|||
// VARBINARY specifies a VARBINARY type.
|
|||
// Properties: 22, IsQuoted, IsBinary.
|
|||
Type_VARBINARY Type = 10262 |
|||
// CHAR specifies a CHAR type.
|
|||
// Properties: 23, IsQuoted, IsText.
|
|||
Type_CHAR Type = 6167 |
|||
// BINARY specifies a BINARY type.
|
|||
// Properties: 24, IsQuoted, IsBinary.
|
|||
Type_BINARY Type = 10264 |
|||
// BIT specifies a BIT type.
|
|||
// Properties: 25, IsQuoted.
|
|||
Type_BIT Type = 2073 |
|||
// JSON specifies a JSON type.
|
|||
// Properties: 30, IsQuoted.
|
|||
Type_JSON Type = 2078 |
|||
) |
|||
|
|||
// BindVariable represents a single bind variable in a Query.
|
|||
type BindVariable struct { |
|||
Type Type `protobuf:"varint,1,opt,name=type,enum=query.Type" json:"type,omitempty"` |
|||
Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` |
|||
// values are set if type is TUPLE.
|
|||
Values []*Value `protobuf:"bytes,3,rep,name=values" json:"values,omitempty"` |
|||
} |
|||
@ -0,0 +1,101 @@ |
|||
package sqltypes |
|||
|
|||
// These bit flags can be used to query on the
|
|||
// common properties of types.
|
|||
const ( |
|||
flagIsIntegral = int(Flag_ISINTEGRAL) |
|||
flagIsUnsigned = int(Flag_ISUNSIGNED) |
|||
flagIsFloat = int(Flag_ISFLOAT) |
|||
flagIsQuoted = int(Flag_ISQUOTED) |
|||
flagIsText = int(Flag_ISTEXT) |
|||
flagIsBinary = int(Flag_ISBINARY) |
|||
) |
|||
|
|||
// IsIntegral returns true if Type is an integral
|
|||
// (signed/unsigned) that can be represented using
|
|||
// up to 64 binary bits.
|
|||
// If you have a Value object, use its member function.
|
|||
func IsIntegral(t Type) bool { |
|||
return int(t)&flagIsIntegral == flagIsIntegral |
|||
} |
|||
|
|||
// IsSigned returns true if Type is a signed integral.
|
|||
// If you have a Value object, use its member function.
|
|||
func IsSigned(t Type) bool { |
|||
return int(t)&(flagIsIntegral|flagIsUnsigned) == flagIsIntegral |
|||
} |
|||
|
|||
// IsUnsigned returns true if Type is an unsigned integral.
|
|||
// Caution: this is not the same as !IsSigned.
|
|||
// If you have a Value object, use its member function.
|
|||
func IsUnsigned(t Type) bool { |
|||
return int(t)&(flagIsIntegral|flagIsUnsigned) == flagIsIntegral|flagIsUnsigned |
|||
} |
|||
|
|||
// IsFloat returns true is Type is a floating point.
|
|||
// If you have a Value object, use its member function.
|
|||
func IsFloat(t Type) bool { |
|||
return int(t)&flagIsFloat == flagIsFloat |
|||
} |
|||
|
|||
// IsQuoted returns true if Type is a quoted text or binary.
|
|||
// If you have a Value object, use its member function.
|
|||
func IsQuoted(t Type) bool { |
|||
return (int(t)&flagIsQuoted == flagIsQuoted) && t != Bit |
|||
} |
|||
|
|||
// IsText returns true if Type is a text.
|
|||
// If you have a Value object, use its member function.
|
|||
func IsText(t Type) bool { |
|||
return int(t)&flagIsText == flagIsText |
|||
} |
|||
|
|||
// IsBinary returns true if Type is a binary.
|
|||
// If you have a Value object, use its member function.
|
|||
func IsBinary(t Type) bool { |
|||
return int(t)&flagIsBinary == flagIsBinary |
|||
} |
|||
|
|||
// isNumber returns true if the type is any type of number.
|
|||
func isNumber(t Type) bool { |
|||
return IsIntegral(t) || IsFloat(t) || t == Decimal |
|||
} |
|||
|
|||
// IsTemporal returns true if Value is time type.
|
|||
func IsTemporal(t Type) bool { |
|||
switch t { |
|||
case Timestamp, Date, Time, Datetime: |
|||
return true |
|||
} |
|||
return false |
|||
} |
|||
|
|||
// Vitess data types. These are idiomatically
|
|||
// named synonyms for the Type values.
|
|||
const ( |
|||
Null = Type_NULL_TYPE |
|||
Int8 = Type_INT8 |
|||
Uint8 = Type_UINT8 |
|||
Int16 = Type_INT16 |
|||
Uint16 = Type_UINT16 |
|||
Int32 = Type_INT32 |
|||
Uint32 = Type_UINT32 |
|||
Int64 = Type_INT64 |
|||
Uint64 = Type_UINT64 |
|||
Float32 = Type_FLOAT32 |
|||
Float64 = Type_FLOAT64 |
|||
Timestamp = Type_TIMESTAMP |
|||
Date = Type_DATE |
|||
Time = Type_TIME |
|||
Datetime = Type_DATETIME |
|||
Year = Type_YEAR |
|||
Decimal = Type_DECIMAL |
|||
Text = Type_TEXT |
|||
Blob = Type_BLOB |
|||
VarChar = Type_VARCHAR |
|||
VarBinary = Type_VARBINARY |
|||
Char = Type_CHAR |
|||
Binary = Type_BINARY |
|||
Bit = Type_BIT |
|||
TypeJSON = Type_JSON |
|||
) |
|||
@ -0,0 +1,30 @@ |
|||
package sqltypes |
|||
|
|||
import ( |
|||
"reflect" |
|||
"unsafe" |
|||
) |
|||
|
|||
// BytesToString casts slice to string without copy
|
|||
func BytesToString(b []byte) (s string) { |
|||
if len(b) == 0 { |
|||
return "" |
|||
} |
|||
|
|||
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) |
|||
sh := reflect.StringHeader{Data: bh.Data, Len: bh.Len} |
|||
|
|||
return *(*string)(unsafe.Pointer(&sh)) |
|||
} |
|||
|
|||
// StringToBytes casts string to slice without copy
|
|||
func StringToBytes(s string) []byte { |
|||
if len(s) == 0 { |
|||
return []byte{} |
|||
} |
|||
|
|||
sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) |
|||
bh := reflect.SliceHeader{Data: sh.Data, Len: sh.Len, Cap: sh.Len} |
|||
|
|||
return *(*[]byte)(unsafe.Pointer(&bh)) |
|||
} |
|||
@ -0,0 +1,355 @@ |
|||
package sqltypes |
|||
|
|||
import ( |
|||
"fmt" |
|||
"strconv" |
|||
"time" |
|||
) |
|||
|
|||
var ( |
|||
// NULL represents the NULL value.
|
|||
NULL = Value{} |
|||
// DontEscape tells you if a character should not be escaped.
|
|||
DontEscape = byte(255) |
|||
nullstr = []byte("null") |
|||
) |
|||
|
|||
type Value struct { |
|||
typ Type |
|||
val []byte |
|||
} |
|||
|
|||
// NewValue builds a Value using typ and val. If the value and typ
|
|||
// don't match, it returns an error.
|
|||
func NewValue(typ Type, val []byte) (v Value, err error) { |
|||
switch { |
|||
case IsSigned(typ): |
|||
if _, err := strconv.ParseInt(string(val), 0, 64); err != nil { |
|||
return NULL, err |
|||
} |
|||
return MakeTrusted(typ, val), nil |
|||
case IsUnsigned(typ): |
|||
if _, err := strconv.ParseUint(string(val), 0, 64); err != nil { |
|||
return NULL, err |
|||
} |
|||
return MakeTrusted(typ, val), nil |
|||
case IsFloat(typ) || typ == Decimal: |
|||
if _, err := strconv.ParseFloat(string(val), 64); err != nil { |
|||
return NULL, err |
|||
} |
|||
return MakeTrusted(typ, val), nil |
|||
case IsQuoted(typ) || typ == Bit || typ == Null: |
|||
return MakeTrusted(typ, val), nil |
|||
} |
|||
// All other types are unsafe or invalid.
|
|||
return NULL, fmt.Errorf("invalid type specified for MakeValue: %v", typ) |
|||
} |
|||
|
|||
// MakeTrusted makes a new Value based on the type.
|
|||
// This function should only be used if you know the value
|
|||
// and type conform to the rules. Every place this function is
|
|||
// called, a comment is needed that explains why it's justified.
|
|||
// Exceptions: The current package and mysql package do not need
|
|||
// comments. Other packages can also use the function to create
|
|||
// VarBinary or VarChar values.
|
|||
func MakeTrusted(typ Type, val []byte) Value { |
|||
|
|||
if typ == Null { |
|||
return NULL |
|||
} |
|||
|
|||
return Value{typ: typ, val: val} |
|||
} |
|||
|
|||
// NewInt64 builds an Int64 Value.
|
|||
func NewInt64(v int64) Value { |
|||
return MakeTrusted(Int64, strconv.AppendInt(nil, v, 10)) |
|||
} |
|||
|
|||
// NewInt32 builds an Int64 Value.
|
|||
func NewInt32(v int32) Value { |
|||
return MakeTrusted(Int32, strconv.AppendInt(nil, int64(v), 10)) |
|||
} |
|||
|
|||
// NewUint64 builds an Uint64 Value.
|
|||
func NewUint64(v uint64) Value { |
|||
return MakeTrusted(Uint64, strconv.AppendUint(nil, v, 10)) |
|||
} |
|||
|
|||
// NewFloat32 builds an Float64 Value.
|
|||
func NewFloat32(v float32) Value { |
|||
return MakeTrusted(Float32, strconv.AppendFloat(nil, float64(v), 'f', -1, 64)) |
|||
} |
|||
|
|||
// NewFloat64 builds an Float64 Value.
|
|||
func NewFloat64(v float64) Value { |
|||
return MakeTrusted(Float64, strconv.AppendFloat(nil, v, 'g', -1, 64)) |
|||
} |
|||
|
|||
// NewVarChar builds a VarChar Value.
|
|||
func NewVarChar(v string) Value { |
|||
return MakeTrusted(VarChar, []byte(v)) |
|||
} |
|||
|
|||
// NewVarBinary builds a VarBinary Value.
|
|||
// The input is a string because it's the most common use case.
|
|||
func NewVarBinary(v string) Value { |
|||
return MakeTrusted(VarBinary, []byte(v)) |
|||
} |
|||
|
|||
// NewIntegral builds an integral type from a string representation.
|
|||
// The type will be Int64 or Uint64. Int64 will be preferred where possible.
|
|||
func NewIntegral(val string) (n Value, err error) { |
|||
signed, err := strconv.ParseInt(val, 0, 64) |
|||
if err == nil { |
|||
return MakeTrusted(Int64, strconv.AppendInt(nil, signed, 10)), nil |
|||
} |
|||
unsigned, err := strconv.ParseUint(val, 0, 64) |
|||
if err != nil { |
|||
return Value{}, err |
|||
} |
|||
return MakeTrusted(Uint64, strconv.AppendUint(nil, unsigned, 10)), nil |
|||
} |
|||
|
|||
// MakeString makes a VarBinary Value.
|
|||
func MakeString(val []byte) Value { |
|||
return MakeTrusted(VarBinary, val) |
|||
} |
|||
|
|||
// BuildValue builds a value from any go type. sqltype.Value is
|
|||
// also allowed.
|
|||
func BuildValue(goval interface{}) (v Value, err error) { |
|||
// Look for the most common types first.
|
|||
switch goval := goval.(type) { |
|||
case nil: |
|||
// no op
|
|||
case []byte: |
|||
v = MakeTrusted(VarBinary, goval) |
|||
case int64: |
|||
v = MakeTrusted(Int64, strconv.AppendInt(nil, int64(goval), 10)) |
|||
case uint64: |
|||
v = MakeTrusted(Uint64, strconv.AppendUint(nil, uint64(goval), 10)) |
|||
case float64: |
|||
v = MakeTrusted(Float64, strconv.AppendFloat(nil, goval, 'f', -1, 64)) |
|||
case int: |
|||
v = MakeTrusted(Int64, strconv.AppendInt(nil, int64(goval), 10)) |
|||
case int8: |
|||
v = MakeTrusted(Int8, strconv.AppendInt(nil, int64(goval), 10)) |
|||
case int16: |
|||
v = MakeTrusted(Int16, strconv.AppendInt(nil, int64(goval), 10)) |
|||
case int32: |
|||
v = MakeTrusted(Int32, strconv.AppendInt(nil, int64(goval), 10)) |
|||
case uint: |
|||
v = MakeTrusted(Uint64, strconv.AppendUint(nil, uint64(goval), 10)) |
|||
case uint8: |
|||
v = MakeTrusted(Uint8, strconv.AppendUint(nil, uint64(goval), 10)) |
|||
case uint16: |
|||
v = MakeTrusted(Uint16, strconv.AppendUint(nil, uint64(goval), 10)) |
|||
case uint32: |
|||
v = MakeTrusted(Uint32, strconv.AppendUint(nil, uint64(goval), 10)) |
|||
case float32: |
|||
v = MakeTrusted(Float32, strconv.AppendFloat(nil, float64(goval), 'f', -1, 64)) |
|||
case string: |
|||
v = MakeTrusted(VarBinary, []byte(goval)) |
|||
case time.Time: |
|||
v = MakeTrusted(Datetime, []byte(goval.Format("2006-01-02 15:04:05"))) |
|||
case Value: |
|||
v = goval |
|||
case *BindVariable: |
|||
return ValueFromBytes(goval.Type, goval.Value) |
|||
default: |
|||
return v, fmt.Errorf("unexpected type %T: %v", goval, goval) |
|||
} |
|||
return v, nil |
|||
} |
|||
|
|||
// BuildConverted is like BuildValue except that it tries to
|
|||
// convert a string or []byte to an integral if the target type
|
|||
// is an integral. We don't perform other implicit conversions
|
|||
// because they're unsafe.
|
|||
func BuildConverted(typ Type, goval interface{}) (v Value, err error) { |
|||
if IsIntegral(typ) { |
|||
switch goval := goval.(type) { |
|||
case []byte: |
|||
return ValueFromBytes(typ, goval) |
|||
case string: |
|||
return ValueFromBytes(typ, []byte(goval)) |
|||
case Value: |
|||
if goval.IsQuoted() { |
|||
return ValueFromBytes(typ, goval.Raw()) |
|||
} |
|||
} |
|||
} |
|||
return BuildValue(goval) |
|||
} |
|||
|
|||
// ValueFromBytes builds a Value using typ and val. It ensures that val
|
|||
// matches the requested type. If type is an integral it's converted to
|
|||
// a canonical form. Otherwise, the original representation is preserved.
|
|||
func ValueFromBytes(typ Type, val []byte) (v Value, err error) { |
|||
switch { |
|||
case IsSigned(typ): |
|||
signed, err := strconv.ParseInt(string(val), 0, 64) |
|||
if err != nil { |
|||
return NULL, err |
|||
} |
|||
v = MakeTrusted(typ, strconv.AppendInt(nil, signed, 10)) |
|||
case IsUnsigned(typ): |
|||
unsigned, err := strconv.ParseUint(string(val), 0, 64) |
|||
if err != nil { |
|||
return NULL, err |
|||
} |
|||
v = MakeTrusted(typ, strconv.AppendUint(nil, unsigned, 10)) |
|||
case IsFloat(typ) || typ == Decimal: |
|||
_, err := strconv.ParseFloat(string(val), 64) |
|||
if err != nil { |
|||
return NULL, err |
|||
} |
|||
// After verification, we preserve the original representation.
|
|||
fallthrough |
|||
default: |
|||
v = MakeTrusted(typ, val) |
|||
} |
|||
return v, nil |
|||
} |
|||
|
|||
// BuildIntegral builds an integral type from a string representation.
|
|||
// The type will be Int64 or Uint64. Int64 will be preferred where possible.
|
|||
func BuildIntegral(val string) (n Value, err error) { |
|||
signed, err := strconv.ParseInt(val, 0, 64) |
|||
if err == nil { |
|||
return MakeTrusted(Int64, strconv.AppendInt(nil, signed, 10)), nil |
|||
} |
|||
unsigned, err := strconv.ParseUint(val, 0, 64) |
|||
if err != nil { |
|||
return Value{}, err |
|||
} |
|||
return MakeTrusted(Uint64, strconv.AppendUint(nil, unsigned, 10)), nil |
|||
} |
|||
|
|||
// Type returns the type of Value.
|
|||
func (v Value) Type() Type { |
|||
return v.typ |
|||
} |
|||
|
|||
// Raw returns the raw bytes. All types are currently implemented as []byte.
|
|||
// You should avoid using this function. If you do, you should treat the
|
|||
// bytes as read-only.
|
|||
func (v Value) Raw() []byte { |
|||
return v.val |
|||
} |
|||
|
|||
// Len returns the length.
|
|||
func (v Value) Len() int { |
|||
return len(v.val) |
|||
} |
|||
|
|||
// Values represents the array of Value.
|
|||
type Values []Value |
|||
|
|||
// Len implements the interface.
|
|||
func (vs Values) Len() int { |
|||
len := 0 |
|||
for _, v := range vs { |
|||
len += v.Len() |
|||
} |
|||
return len |
|||
} |
|||
|
|||
// String returns the raw value as a string.
|
|||
func (v Value) String() string { |
|||
return BytesToString(v.val) |
|||
} |
|||
|
|||
// ToNative converts Value to a native go type.
|
|||
// This does not work for sqltypes.Tuple. The function
|
|||
// panics if there are inconsistencies.
|
|||
func (v Value) ToNative() interface{} { |
|||
var out interface{} |
|||
var err error |
|||
switch { |
|||
case v.typ == Null: |
|||
// no-op
|
|||
case IsSigned(v.typ): |
|||
out, err = v.ParseInt64() |
|||
case IsUnsigned(v.typ): |
|||
out, err = v.ParseUint64() |
|||
case IsFloat(v.typ): |
|||
out, err = v.ParseFloat64() |
|||
default: |
|||
out = v.val |
|||
} |
|||
if err != nil { |
|||
panic(err) |
|||
} |
|||
return out |
|||
} |
|||
|
|||
// ParseInt64 will parse a Value into an int64. It does
|
|||
// not check the type.
|
|||
func (v Value) ParseInt64() (val int64, err error) { |
|||
return strconv.ParseInt(v.String(), 10, 64) |
|||
} |
|||
|
|||
// ParseUint64 will parse a Value into a uint64. It does
|
|||
// not check the type.
|
|||
func (v Value) ParseUint64() (val uint64, err error) { |
|||
return strconv.ParseUint(v.String(), 10, 64) |
|||
} |
|||
|
|||
// ParseFloat64 will parse a Value into an float64. It does
|
|||
// not check the type.
|
|||
func (v Value) ParseFloat64() (val float64, err error) { |
|||
return strconv.ParseFloat(v.String(), 64) |
|||
} |
|||
|
|||
// IsNull returns true if Value is null.
|
|||
func (v Value) IsNull() bool { |
|||
return v.typ == Null |
|||
} |
|||
|
|||
// IsIntegral returns true if Value is an integral.
|
|||
func (v Value) IsIntegral() bool { |
|||
return IsIntegral(v.typ) |
|||
} |
|||
|
|||
// IsSigned returns true if Value is a signed integral.
|
|||
func (v Value) IsSigned() bool { |
|||
return IsSigned(v.typ) |
|||
} |
|||
|
|||
// IsUnsigned returns true if Value is an unsigned integral.
|
|||
func (v Value) IsUnsigned() bool { |
|||
return IsUnsigned(v.typ) |
|||
} |
|||
|
|||
// IsFloat returns true if Value is a float.
|
|||
func (v Value) IsFloat() bool { |
|||
return IsFloat(v.typ) |
|||
} |
|||
|
|||
// IsQuoted returns true if Value must be SQL-quoted.
|
|||
func (v Value) IsQuoted() bool { |
|||
return IsQuoted(v.typ) |
|||
} |
|||
|
|||
// IsText returns true if Value is a collatable text.
|
|||
func (v Value) IsText() bool { |
|||
return IsText(v.typ) |
|||
} |
|||
|
|||
// IsBinary returns true if Value is binary.
|
|||
func (v Value) IsBinary() bool { |
|||
return IsBinary(v.typ) |
|||
} |
|||
|
|||
// IsTemporal returns true if Value is time type.
|
|||
func (v Value) IsTemporal() bool { |
|||
return IsTemporal(v.typ) |
|||
} |
|||
|
|||
// ToString returns the value as MySQL would return it as string.
|
|||
// If the value is not convertible like in the case of Expression, it returns nil.
|
|||
func (v Value) ToString() string { |
|||
return BytesToString(v.val) |
|||
} |
|||
@ -0,0 +1,69 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/operation" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/query/json" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
|||
"github.com/tidwall/gjson" |
|||
) |
|||
|
|||
func (vs *VolumeServer) Query(req *volume_server_pb.QueryRequest, stream volume_server_pb.VolumeServer_QueryServer) error { |
|||
|
|||
for _, fid := range req.FromFileIds { |
|||
|
|||
vid, id_cookie, err := operation.ParseFileId(fid) |
|||
if err != nil { |
|||
glog.V(0).Infof("volume query failed to parse fid %s: %v", fid, err) |
|||
return err |
|||
} |
|||
|
|||
n := new(needle.Needle) |
|||
volumeId, _ := needle.NewVolumeId(vid) |
|||
n.ParsePath(id_cookie) |
|||
|
|||
cookie := n.Cookie |
|||
if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil { |
|||
glog.V(0).Infof("volume query failed to read fid %s: %v", fid, err) |
|||
return err |
|||
} |
|||
|
|||
if n.Cookie != cookie { |
|||
glog.V(0).Infof("volume query failed to read fid cookie %s: %v", fid, err) |
|||
return err |
|||
} |
|||
|
|||
if req.InputSerialization.CsvInput != nil { |
|||
|
|||
} |
|||
|
|||
if req.InputSerialization.JsonInput != nil { |
|||
|
|||
stripe := &volume_server_pb.QueriedStripe{ |
|||
Records: nil, |
|||
} |
|||
|
|||
filter := json.Query{ |
|||
Field: req.Filter.Field, |
|||
Op: req.Filter.Operand, |
|||
Value: req.Filter.Value, |
|||
} |
|||
gjson.ForEachLine(string(n.Data), func(line gjson.Result) bool { |
|||
passedFilter, values := json.QueryJson(line.Raw, req.Selections, filter) |
|||
if !passedFilter { |
|||
return true |
|||
} |
|||
stripe.Records = json.ToJson(stripe.Records, req.Selections, values) |
|||
return true |
|||
}) |
|||
err = stream.Send(stripe) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
return nil |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue