scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into fix-writable

Browse source
This commit is contained in:
Kuba Podgórski 2020-09-15 19:14:24 +02:00 • committed by GitHub
commit dcc237413e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 81 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
executor.go
View file

@ -5370,18 +5370,11 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
return nil, newNotFoundError(ErrFieldNotFound, v)
}
datatype, err := field.Datatype()
if err != nil {
return nil, errors.Wrapf(err, "field %s", v)
}
fields[i] = ExtractedTableField{
Name: v,
Type: datatype,
}
var mapper fieldMapper
var datatype string
switch typ := field.Type(); typ {
case FieldTypeBool:
datatype = "bool"
mapper = func(ids []uint64) (_ interface{}, err error) {
switch len(ids) {
case 0:
@ -5401,6 +5394,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
case FieldTypeSet, FieldTypeTime:
if field.Keys() {
datatype = "[]string"
translations, err := e.preTranslateMatrixSet(result, uint(i), field)
if err != nil {
return nil, errors.Wrapf(err, "translating IDs of field %q", v)
@ -5413,6 +5407,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
return keys, nil
}
} else {
datatype = "[]uint64"
mapper = func(ids []uint64) (interface{}, error) {
if ids == nil {
ids = []uint64{}
@ -5422,6 +5417,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
case FieldTypeMutex:
if field.Keys() {
datatype = "string"
translations, err := e.preTranslateMatrixSet(result, uint(i), field)
if err != nil {
return nil, errors.Wrapf(err, "translating IDs of field %q", v)
@ -5437,6 +5433,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
}
} else {
datatype = "uint64"
mapper = func(ids []uint64) (_ interface{}, err error) {
switch len(ids) {
case 0:
@ -5451,6 +5448,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
case FieldTypeInt:
if fi := field.ForeignIndex(); fi != "" {
if field.Keys() {
datatype = "string"
ids := make(map[uint64]struct{}, len(result.Columns))
for _, col := range result.Columns {
for _, v := range col.Rows[i] {
@ -5472,6 +5470,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
}
} else {
datatype = "uint64"
mapper = func(ids []uint64) (interface{}, error) {
switch len(ids) {
case 0:
@ -5484,6 +5483,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
}
} else {
datatype = "int64"
mapper = func(ids []uint64) (interface{}, error) {
switch len(ids) {
case 0:
@ -5496,6 +5496,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
}
}
case FieldTypeDecimal:
datatype = "decimal"
scale := field.Options().Scale
mapper = func(ids []uint64) (_ interface{}, err error) {
switch len(ids) {
@ -5511,6 +5512,10 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index
return nil, errors.Errorf("field type %q not yet supported", typ)
}
mappers[i] = mapper
fields[i] = ExtractedTableField{
Name: v,
Type: datatype,
}
}
var translateCol func(uint64) (KeyOrID, error)

80
executor_test.go
View file

@ -4414,11 +4414,7 @@ func TestExecutor_Execute_Extract(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
set := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "set")
dtSet, err := set.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "set")
c.ImportBits(t, "i", "set", [][2]uint64{
{0, 1},
{0, 2},
@ -4429,88 +4425,56 @@ func TestExecutor_Execute_Extract(t *testing.T) {
})
c.Query(t, "i", fmt.Sprintf("Clear(%d, set=5)", ShardWidth))
keyset := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keyset", pilosa.OptFieldKeys())
dtKeyset, err := keyset.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keyset", pilosa.OptFieldKeys())
c.Query(t, "i", `
Set(0, keyset="h")
Set(1, keyset="xyzzy")
Set(0, keyset="plugh")
`)
mutex := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "mutex", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000))
dtMutex, err := mutex.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "mutex", pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000))
c.ImportBits(t, "i", "mutex", [][2]uint64{
{0, 1},
{0, 2},
{4, 4 * ShardWidth},
})
keymutex := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keymutex", pilosa.OptFieldKeys(), pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000))
dtKeyMutex, err := keymutex.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keymutex", pilosa.OptFieldKeys(), pilosa.OptFieldTypeMutex(pilosa.CacheTypeRanked, 5000))
c.Query(t, "i", `
Set(0, keymutex="h")
Set(1, keymutex="xyzzy")
Set(3, keymutex="plugh")
`)
tm := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH"))
dtTm, err := tm.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH"))
c.Query(t, "i", `
Set(0, time=1, 2016-01-01T00:00)
Set(1, time=2, 2017-01-01T00:00)
Set(3, time=3, 2018-01-01T00:00)
`)
keytm := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH"))
dtKeyTm, err := keytm.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH"))
c.Query(t, "i", `
Set(0, keytime="h", 2016-01-01T00:00)
Set(1, keytime="xyzzy", 2017-01-01T00:00)
Set(0, keytime="plugh", 2018-01-01T00:00)
`)
bsiInt := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "bsint", pilosa.OptFieldTypeInt(-100, 100))
dtBsiInt, err := bsiInt.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "bsint", pilosa.OptFieldTypeInt(-100, 100))
c.Query(t, "i", `
Set(0, bsint=1)
Set(1, bsint=-1)
Set(3, bsint=2)
`)
bsidecimal := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "bsidecimal", pilosa.OptFieldTypeDecimal(2))
dtBsiDecimal, err := bsidecimal.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "bsidecimal", pilosa.OptFieldTypeDecimal(2))
c.Query(t, "i", `
Set(0, bsidecimal=0.01)
Set(1, bsidecimal=1.00)
Set(3, bsidecimal=-1.01)
`)
boolean := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "bool", pilosa.OptFieldTypeBool())
dtBoolean, err := boolean.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "bool", pilosa.OptFieldTypeBool())
c.Query(t, "i", `
Set(0, bool=true)
Set(1, bool=false)
@ -4523,39 +4487,39 @@ func TestExecutor_Execute_Extract(t *testing.T) {
Fields: []pilosa.ExtractedTableField{
{
Name: "set",
Type: dtSet,
Type: "[]uint64",
},
{
Name: "keyset",
Type: dtKeyset,
Type: "[]string",
},
{
Name: "mutex",
Type: dtMutex,
Type: "uint64",
},
{
Name: "keymutex",
Type: dtKeyMutex,
Type: "string",
},
{
Name: "time",
Type: dtTm,
Type: "[]uint64",
},
{
Name: "keytime",
Type: dtKeyTm,
Type: "[]string",
},
{
Name: "bsint",
Type: dtBsiInt,
Type: "int64",
},
{
Name: "bsidecimal",
Type: dtBsiDecimal,
Type: "decimal",
},
{
Name: "bool",
Type: dtBoolean,
Type: "bool",
},
},
Columns: []pilosa.ExtractedTableColumn{
@ -4680,11 +4644,7 @@ func TestExecutor_Execute_Extract_Keyed(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
set := c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true, Keys: true}, "set")
dtSet, err := set.Datatype()
if err != nil {
t.Fatal(err)
}
c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true, Keys: true}, "set")
c.Query(t, "i", `
Set("h", set=1)
Set("h", set=2)
@ -4699,7 +4659,7 @@ func TestExecutor_Execute_Extract_Keyed(t *testing.T) {
Fields: []pilosa.ExtractedTableField{
{
Name: "set",
Type: dtSet,
Type: "[]uint64",
},
},
Columns: []pilosa.ExtractedTableColumn{

30
field.go
View file

@ -1353,36 +1353,6 @@ func (f *Field) ClearBit(tx Tx, rowID, colID uint64) (changed bool, err error) {
return changed, nil
}
// Datatype returns a useful data type (string,
// uint64, bool, etc.) based on the field type.
func (f *Field) Datatype() (string, error) {
switch t := f.Type(); t {
case "set":
if f.Keys() {
return "[]string", nil
}
return "[]uint64", nil
case "mutex":
if f.Keys() {
return "string", nil
}
return "uint64", nil
case "int":
if f.Keys() {
return "string", nil
}
return "int64", nil
case "decimal":
return "decimal", nil
case "bool":
return "bool", nil
case "time":
return "int64", nil // TODO: this is a placeholder
default:
return "", fmt.Errorf("unimplemented field Datatype: %s", t)
}
}
func groupCompare(a, b string, offset int) (lt, eq bool) {
if len(a) > offset {
a = a[:offset]

37
server/grpc.go
View file

@ -469,7 +469,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
{Name: "_id", Datatype: "uint64"},
}
for _, field := range fields {
fdt, err := field.Datatype()
fdt := fieldDataType(field)
if err != nil {
return errors.Wrapf(err, "field %s", field.Name())
}
@ -757,7 +757,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
{Name: "_id", Datatype: "string"},
}
for _, field := range fields {
fdt, err := field.Datatype()
fdt := fieldDataType(field)
if err != nil {
return errors.Wrapf(err, "field %s", field.Name())
}
@ -1036,6 +1036,39 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
return nil
}
// fieldDataType returns a useful data type (string,
// uint64, bool, etc.) based on the Pilosa field type.
// DO NOT USE THIS IN FUTURE CODE.
// This remains only for backwards-compatability within inspect.
// It does not produce sane results in all scenarios.
func fieldDataType(f *pilosa.Field) string {
switch f.Type() {
case "set":
if f.Keys() {
return "[]string"
}
return "[]uint64"
case "mutex":
if f.Keys() {
return "string"
}
return "uint64"
case "int":
if f.Keys() {
return "string"
}
return "int64"
case "decimal":
return "decimal"
case "bool":
return "bool"
case "time":
return "int64" // TODO: this is a placeholder
default:
panic(fmt.Sprintf("unimplemented fieldDataType: %s", f.Type()))
}
}
type grpcServer struct {
api *pilosa.API
mu sync.Mutex

8
server/grpc_test.go
View file

@ -726,10 +726,10 @@ func TestQuerySQLUnary(t *testing.T) {
{"Type", "string"},
},
rows: []row{
{[]columnResponse{"age", "int64"}},
{[]columnResponse{"color", "[]string"}},
{[]columnResponse{"height", "int64"}},
{[]columnResponse{"score", "int64"}},
{[]columnResponse{"age", "int"}},
{[]columnResponse{"color", "keyed-set"}},
{[]columnResponse{"height", "int"}},
{[]columnResponse{"score", "int"}},
},
},
eq: equal,

12
sql/show.go
View file

@ -89,10 +89,14 @@ func (s *ShowHandler) execShowFields(ctx context.Context, showStmt *sqlparser.Sh
continue
}
dt, err := f.Datatype()
if err != nil {
return nil, errors.Wrapf(err, "field %s", f.Name())
typeName := f.Type()
if f.Keys() {
typeName = "keyed-" + typeName
}
if f.ForeignIndex() != "" {
typeName = "foreign-" + typeName
}
result = append(result, pproto.RowResponse{
Headers: []*pproto.ColumnInfo{
{Name: "Field", Datatype: "string"},
@ -100,7 +104,7 @@ func (s *ShowHandler) execShowFields(ctx context.Context, showStmt *sqlparser.Sh
},
Columns: []*pproto.ColumnResponse{
{ColumnVal: &pproto.ColumnResponse_StringVal{StringVal: f.Name()}},
{ColumnVal: &pproto.ColumnResponse_StringVal{StringVal: dt}},
{ColumnVal: &pproto.ColumnResponse_StringVal{StringVal: typeName}},
},
})
}