From 6f443f447ae3eb9961391cda16f2c9a708293017 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 11 Apr 2017 13:51:14 -0500 Subject: [PATCH] add signed int support this affects the scanner/parser and makes it produce signed ints rather than uints, and so it affects all code that is expecting unsighed ints to come out of the parser. --- attr.go | 24 ++++----- attr_test.go | 6 +-- executor.go | 118 +++++++++++++++++++++++++++-------------- executor_test.go | 10 ++-- fragment_test.go | 2 +- handler_test.go | 4 +- internal/private.pb.go | 2 +- internal/public.pb.go | 90 +++++++++++++++---------------- internal/public.proto | 2 +- pql/ast.go | 38 +++++++++++++ pql/parser.go | 4 +- pql/parser_test.go | 28 ++++++++-- pql/scanner.go | 8 +-- server/server_test.go | 24 +++++++-- 14 files changed, 238 insertions(+), 122 deletions(-) diff --git a/attr.go b/attr.go index 804b51a34..f619dda9f 100644 --- a/attr.go +++ b/attr.go @@ -20,7 +20,7 @@ const AttrBlockSize = 100 // Attribute data type enum. const ( AttrTypeString = 1 - AttrTypeUint = 2 + AttrTypeInt = 2 AttrTypeBool = 3 AttrTypeFloat = 4 ) @@ -263,12 +263,12 @@ func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string switch v := v.(type) { case int: - attr[k] = uint64(v) + attr[k] = int64(v) case uint: - attr[k] = uint64(v) - case int64: - attr[k] = uint64(v) - case string, uint64, bool, float64: + attr[k] = int64(v) + case uint64: + attr[k] = int64(v) + case string, int64, bool, float64: attr[k] = v default: return nil, fmt.Errorf("invalid attr type: %T", v) @@ -320,11 +320,11 @@ func encodeAttr(key string, value interface{}) *internal.Attr { pb.Type = AttrTypeFloat pb.FloatValue = value case uint64: - pb.Type = AttrTypeUint - pb.UintValue = value + pb.Type = AttrTypeInt + pb.IntValue = int64(value) case int64: - pb.Type = AttrTypeUint - pb.UintValue = uint64(value) + pb.Type = AttrTypeInt + pb.IntValue = value case bool: pb.Type = AttrTypeBool pb.BoolValue = value @@ -337,8 +337,8 @@ func decodeAttr(attr *internal.Attr) (key string, value interface{}) { switch attr.Type { case AttrTypeString: return attr.Key, attr.StringValue - case AttrTypeUint: - return attr.Key, attr.UintValue + case AttrTypeInt: + return attr.Key, attr.IntValue case AttrTypeBool: return attr.Key, attr.BoolValue case AttrTypeFloat: diff --git a/attr_test.go b/attr_test.go index e69b8df2e..f026f6244 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,7 +15,7 @@ func TestAttrStore_Attrs(t *testing.T) { defer s.Close() // Set attributes. - if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil { + if err := s.SetAttrs(1, map[string]interface{}{"A": 100, "C": -27}); err != nil { t.Fatal(err) } else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil { t.Fatal(err) @@ -26,14 +26,14 @@ func TestAttrStore_Attrs(t *testing.T) { // Retrieve attributes for profile #1. if m, err := s.Attrs(1); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(100), "B": "VALUE"}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) { t.Fatalf("unexpected attrs(1): %#v", m) } // Retrieve attributes for profile #2. if m, err := s.Attrs(2); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(200)}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) { t.Fatalf("unexpected attrs(2): %#v", m) } } diff --git a/executor.go b/executor.go index 788e8fe5e..ef510ffb9 100644 --- a/executor.go +++ b/executor.go @@ -117,12 +117,12 @@ func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slic func (e *Executor) validateCallArgs(c *pql.Call) error { if _, ok := c.Args["ids"]; ok { switch v := c.Args["ids"].(type) { - case []uint64: + case []int64, []uint64: // noop case []interface{}: - b := make([]uint64, len(v), len(v)) + b := make([]int64, len(v), len(v)) for i := range v { - b[i] = v[i].(uint64) + b[i] = v[i].(int64) } c.Args["ids"] = b default: @@ -159,20 +159,26 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call // If the row label is used then return bitmap attributes. bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { + d := e.Index.DB(db) if d != nil { columnLabel := d.ColumnLabel() - if columnID, ok := c.Args[columnLabel].(uint64); ok { + if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { attrs, err := d.ProfileAttrStore().Attrs(columnID) if err != nil { return nil, err } bm.Attrs = attrs + } else if err != nil { + return nil, err } else { frame, _ := c.Args["frame"].(string) if fr := d.Frame(frame); fr != nil { rowLabel := fr.RowLabel() - rowID, _ := c.Args[rowLabel].(uint64) + rowID, _, err := c.UintArg(rowLabel) + if err != nil { + return nil, err + } attrs, err := fr.BitmapAttrStore().Attrs(rowID) if err != nil { return nil, err @@ -208,10 +214,13 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { - bitmapIDs, _ := c.Args["ids"].([]uint64) - var n uint64 - if nval, ok := c.Args["n"]; ok { - n = nval.(uint64) + bitmapIDs, _, err := c.UintSliceArg("ids") + if err != nil { + return nil, fmt.Errorf("executeTopN: %v", err) + } + n, _, err := c.UintArg("n") + if err != nil { + return nil, fmt.Errorf("executeTopN: %v", err) } // Execute original query. @@ -270,12 +279,25 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, slice uint64) ([]Pair, error) { frame, _ := c.Args["frame"].(string) - n, _ := c.Args["n"].(uint64) + n, _, err := c.UintArg("n") + if err != nil { + return nil, fmt.Errorf("executeTopNSlice: %v", err) + } field, _ := c.Args["field"].(string) - bitmapIDs, _ := c.Args["ids"].([]uint64) - minThreshold, _ := c.Args["threshold"].(uint64) + bitmapIDs, _, err := c.UintSliceArg("ids") + if err != nil { + return nil, fmt.Errorf("executeTopNSlice: %v", err) + } + minThreshold, _, err := c.UintArg("threshold") + if err != nil { + return nil, fmt.Errorf("executeTopNSlice: %v", err) + } filters, _ := c.Args["filters"].([]interface{}) - tanimotoThreshold, _ := c.Args["tanimotoThreshold"].(uint64) + tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold") + if err != nil { + return nil, fmt.Errorf("executeTopNSlice: %v", err) + } + // Retrieve bitmap used to intersect. var src *Bitmap if len(c.Children) == 1 { @@ -358,8 +380,11 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal rowLabel := f.RowLabel() // Return an error if both the row and column label are specified. - rowID, rowOK := c.Args[rowLabel].(uint64) - columnID, columnOK := c.Args[columnLabel].(uint64) + rowID, rowOK, rowErr := c.UintArg(rowLabel) + columnID, columnOK, columnErr := c.UintArg(columnLabel) + if rowErr != nil || columnErr != nil { + return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) + } if rowOK && columnOK { return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel) } else if !rowOK && !columnOK { @@ -420,7 +445,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call rowLabel := f.RowLabel() // Read row id. - rowID, _ := c.Args[rowLabel].(uint64) + rowID, _, err := c.UintArg(rowLabel) // TODO: why are we ignoring missing rowID? + if err != nil { + return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) + } // Parse start time. startTimeStr, ok := c.Args["start"].(string) @@ -534,14 +562,18 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, rowLabel := f.RowLabel() // Read fields using labels. - rowID, ok := c.Args[rowLabel].(uint64) - if !ok { - return false, fmt.Errorf("ClearBit() field required: %s", rowLabel) + rowID, ok, err := c.UintArg(rowLabel) + if err != nil { + return false, fmt.Errorf("reading ClearBit() row: %v", err) + } else if !ok { + return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel) } - colID, ok := c.Args[columnLabel].(uint64) - if !ok { - return false, fmt.Errorf("ClearBit() field required: %s", columnLabel) + colID, ok, err := c.UintArg(columnLabel) + if err != nil { + return false, fmt.Errorf("reading ClearBit() column: %v", err) + } else if !ok { + return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) } // Clear bits for each view. @@ -624,14 +656,18 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op rowLabel := f.RowLabel() // Read fields using labels. - rowID, ok := c.Args[rowLabel].(uint64) - if !ok { - return false, fmt.Errorf("SetBit() field required: %s", rowLabel) + rowID, ok, err := c.UintArg(rowLabel) + if err != nil { + return false, fmt.Errorf("reading SetBit() row: %v", err) + } else if !ok { + return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel) } - colID, ok := c.Args[columnLabel].(uint64) - if !ok { - return false, fmt.Errorf("SetBit() field required: %s", columnLabel) + colID, ok, err := c.UintArg(columnLabel) + if err != nil { + return false, fmt.Errorf("reading SetBit() column: %v", err) + } else if !ok { + return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel) } var timestamp *time.Time @@ -718,9 +754,11 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. rowLabel := frame.RowLabel() // Parse labels. - rowID, ok := c.Args[rowLabel].(uint64) - if !ok { - return fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel) + rowID, ok, err := c.UintArg(rowLabel) + if err != nil { + return fmt.Errorf("reading SetBitmapAttrs() row: %v", err) + } else if !ok { + return fmt.Errorf("SetBitmapAttrs() row field '%v' required.", rowLabel) } // Copy args and remove reserved fields. @@ -775,9 +813,11 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal } rowLabel := f.RowLabel() - rowID, ok := c.Args[rowLabel].(uint64) - if !ok { - return nil, fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel) + rowID, ok, err := c.UintArg(rowLabel) + if err != nil { + return nil, fmt.Errorf("reading SetBitmapAttrs() row: %v", rowLabel) + } else if !ok { + return nil, fmt.Errorf("SetBitmapAttrs row field '%v' required.", rowLabel) } // Copy args and remove reserved fields. @@ -852,13 +892,13 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql } var colName string - id, ok := c.Args["id"].(uint64) - if !ok { + id, okID, errID := c.UintArg("id") + if errID != nil || !okID { // Retrieve columnLabel columnLabel := d.columnLabel - col, ok := c.Args[columnLabel].(uint64) - if !ok { - return errors.New("SetProfileAttrs() id required") + col, okCol, errCol := c.UintArg(columnLabel) + if errCol != nil || !okCol { + return fmt.Errorf("reading SetProfileAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) } id = col colName = columnLabel diff --git a/executor_test.go b/executor_test.go index d7c661765..be005e732 100644 --- a/executor_test.go +++ b/executor_test.go @@ -41,7 +41,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) { + } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) @@ -72,7 +72,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) { + } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) @@ -251,7 +251,7 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { f := idx.Frame("d", "f") if m, err := f.BitmapAttrStore().Attrs(10); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": uint64(123), "bat": true}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { t.Fatalf("unexpected bitmap attr: %#v", m) } } @@ -269,7 +269,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0) // Execute query. @@ -382,7 +382,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(idx.Index, NewCluster(1)) diff --git a/fragment_test.go b/fragment_test.go index 199ba0a06..b915bd449 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -177,7 +177,7 @@ func TestFragment_Top_Filter(t *testing.T) { if pairs, err := f.Top(pilosa.TopOptions{ N: 2, FilterField: "x", - FilterValues: []interface{}{uint64(10), uint64(15), uint64(20)}, + FilterValues: []interface{}{int64(10), int64(15), int64(20)}, }); err != nil { t.Fatal(err) } else if len(pairs) != 2 { diff --git a/handler_test.go b/handler_test.go index 95c77211a..200deb0b3 100644 --- a/handler_test.go +++ b/handler_test.go @@ -312,7 +312,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) @@ -369,7 +369,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) diff --git a/internal/private.pb.go b/internal/private.pb.go index ed2e6d41f..1ffa98885 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -1617,7 +1617,7 @@ func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ // 406 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xd1, 0x8a, 0xd3, 0x40, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x92, 0xd1, 0x8a, 0xd3, 0x40, 0x14, 0x86, 0x9d, 0x36, 0x95, 0xf6, 0x2c, 0x86, 0xee, 0xb0, 0x17, 0x61, 0x59, 0x42, 0x18, 0x50, 0x8a, 0x17, 0xbd, 0xd0, 0x1b, 0x11, 0xaf, 0xb2, 0xad, 0x6c, 0x41, 0xc1, 0x1d, 0x17, 0xef, 0xa7, 0xf5, 0x88, 0xa1, 0x93, 0x99, 0x38, 0x99, 0x74, 0x37, 0xcf, 0xe0, 0x0b, 0x08, 0xbe, 0x90, 0x97, diff --git a/internal/public.pb.go b/internal/public.pb.go index 82645d9ee..904ed879e 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -98,7 +98,7 @@ type Attr struct { Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` - UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,proto3" json:"UintValue,omitempty"` + IntValue int64 `protobuf:"varint,4,opt,name=IntValue,proto3" json:"IntValue,omitempty"` BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` FloatValue float64 `protobuf:"fixed64,6,opt,name=FloatValue,proto3" json:"FloatValue,omitempty"` } @@ -390,10 +390,10 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.StringValue))) i += copy(dAtA[i:], m.StringValue) } - if m.UintValue != 0 { + if m.IntValue != 0 { dAtA[i] = 0x20 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.UintValue)) + i = encodeVarintPublic(dAtA, i, uint64(m.IntValue)) } if m.BoolValue { dAtA[i] = 0x28 @@ -808,8 +808,8 @@ func (m *Attr) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - if m.UintValue != 0 { - n += 1 + sovPublic(uint64(m.UintValue)) + if m.IntValue != 0 { + n += 1 + sovPublic(uint64(m.IntValue)) } if m.BoolValue { n += 2 @@ -1504,9 +1504,9 @@ func (m *Attr) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field UintValue", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field IntValue", wireType) } - m.UintValue = 0 + m.IntValue = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1516,7 +1516,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.UintValue |= (uint64(b) & 0x7F) << shift + m.IntValue |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -2615,41 +2615,41 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 572 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd4, 0x3c, - 0x14, 0xfd, 0x3c, 0xc9, 0xfc, 0xdd, 0x69, 0xab, 0x7e, 0x16, 0xa0, 0x08, 0xa1, 0x51, 0x14, 0xb1, - 0xc8, 0x86, 0xa9, 0x54, 0x1e, 0x00, 0x91, 0x4e, 0x2b, 0x8d, 0x10, 0x55, 0xeb, 0x16, 0x76, 0x2c, - 0xd2, 0x62, 0x8a, 0xa5, 0xfc, 0x61, 0x3b, 0x8b, 0x59, 0xb2, 0xe0, 0x1d, 0x10, 0x6f, 0x00, 0x4f, - 0xc2, 0x92, 0x47, 0x40, 0xc3, 0x8b, 0xa0, 0xeb, 0x9f, 0x24, 0xab, 0x8a, 0x9d, 0xcf, 0x39, 0x73, - 0x9d, 0x7b, 0x7c, 0xee, 0x1d, 0xd8, 0x6b, 0xda, 0x9b, 0x42, 0xdc, 0xae, 0x1a, 0x59, 0xeb, 0x9a, - 0xce, 0x44, 0xa5, 0xb9, 0xac, 0xf2, 0x22, 0xc9, 0x60, 0x92, 0x09, 0x5d, 0xe6, 0x0d, 0xa5, 0x10, - 0x66, 0x42, 0xab, 0x88, 0xc4, 0x41, 0x1a, 0x32, 0x73, 0xa6, 0x4f, 0x61, 0xfc, 0x52, 0x6b, 0xa9, - 0xa2, 0x51, 0x1c, 0xa4, 0x8b, 0xe3, 0x83, 0x95, 0xaf, 0x5b, 0x21, 0xcd, 0xac, 0x98, 0xac, 0x20, - 0xbc, 0xc8, 0x85, 0xa4, 0x87, 0x10, 0xbc, 0xe2, 0xdb, 0x88, 0xc4, 0x24, 0x0d, 0x19, 0x1e, 0xe9, - 0x03, 0x18, 0x9f, 0xd4, 0x6d, 0xa5, 0xa3, 0x91, 0xe1, 0x2c, 0x48, 0xde, 0x41, 0x90, 0x09, 0x4d, - 0x1f, 0xc3, 0xcc, 0x7e, 0x7a, 0xb3, 0x76, 0x35, 0x1d, 0xa6, 0x4f, 0x60, 0x7e, 0x21, 0xeb, 0x0f, - 0xa2, 0xe0, 0x9b, 0xb5, 0x2b, 0xee, 0x09, 0x54, 0xaf, 0x45, 0xc9, 0x95, 0xce, 0xcb, 0x26, 0x0a, - 0x62, 0x92, 0x06, 0xac, 0x27, 0x92, 0x17, 0x30, 0x75, 0x3f, 0xa5, 0x07, 0x30, 0xea, 0x2e, 0x1f, - 0x6d, 0xd6, 0xff, 0xe8, 0xe7, 0x07, 0x81, 0x10, 0x4f, 0x43, 0x43, 0x73, 0x6b, 0x88, 0x42, 0x78, - 0xbd, 0x6d, 0xb8, 0x6b, 0xc9, 0x9c, 0x69, 0x0c, 0x8b, 0x2b, 0x2d, 0x45, 0x75, 0xf7, 0x36, 0x2f, - 0x5a, 0x6e, 0xfa, 0x99, 0xb3, 0x21, 0x85, 0xfd, 0xbe, 0x11, 0x95, 0xb6, 0x7a, 0x68, 0xdd, 0x74, - 0x04, 0xaa, 0x59, 0x5d, 0x17, 0x56, 0x1d, 0xc7, 0x24, 0x9d, 0xb1, 0x9e, 0xa0, 0x4b, 0x80, 0xb3, - 0xa2, 0xce, 0x5d, 0xf1, 0x24, 0x26, 0x29, 0x61, 0x03, 0x26, 0x39, 0x82, 0x29, 0xf6, 0xfa, 0x3a, - 0x6f, 0x7a, 0x77, 0xe4, 0x3e, 0x77, 0xdf, 0x08, 0xec, 0x5d, 0xb6, 0x5c, 0x6e, 0x19, 0xff, 0xd4, - 0x72, 0xa5, 0xf1, 0x91, 0xd6, 0x99, 0x33, 0x39, 0x5a, 0x67, 0x18, 0x9a, 0xd1, 0x8d, 0xc9, 0x39, - 0xb3, 0x80, 0x3e, 0x82, 0xc9, 0x55, 0x21, 0x6e, 0xb9, 0x8a, 0x02, 0x33, 0x20, 0x0e, 0x61, 0x8a, - 0xee, 0xb5, 0x95, 0xb1, 0x36, 0x63, 0x1d, 0xa6, 0x11, 0x4c, 0x2f, 0xdb, 0xbc, 0xd2, 0x6d, 0x69, - 0x7c, 0xcd, 0x99, 0x87, 0x78, 0x1b, 0xe3, 0x65, 0xad, 0xad, 0xa3, 0x19, 0x73, 0x28, 0xf9, 0x4c, - 0x60, 0xdf, 0x35, 0xa7, 0x9a, 0xba, 0x52, 0x1c, 0x33, 0x38, 0x95, 0xd2, 0x67, 0x70, 0x2a, 0x25, - 0x3d, 0x82, 0x29, 0xe3, 0xaa, 0x2d, 0xb4, 0x8f, 0xf1, 0x61, 0x6f, 0xd4, 0xd7, 0xb6, 0x85, 0x66, - 0xfe, 0x57, 0xf4, 0xd9, 0xa0, 0xc5, 0xc0, 0x54, 0xfc, 0xdf, 0x57, 0x38, 0xa5, 0xef, 0x3a, 0xf9, - 0x42, 0x60, 0x31, 0xb8, 0x87, 0xa6, 0x7e, 0x45, 0x4c, 0x13, 0x8b, 0xe3, 0xc3, 0xbe, 0xd8, 0xf2, - 0xcc, 0xaf, 0xd0, 0x1e, 0x90, 0x73, 0x37, 0x1a, 0xe4, 0x1c, 0xe3, 0xc0, 0xb5, 0xf0, 0xdf, 0x1c, - 0xc4, 0x81, 0x34, 0xb3, 0x22, 0xbe, 0xd1, 0xc9, 0xc7, 0xbc, 0xba, 0xe3, 0xef, 0xdd, 0xf3, 0x79, - 0x98, 0x7c, 0x27, 0xb0, 0xbf, 0x29, 0x9b, 0x5a, 0xea, 0x7b, 0x92, 0x3a, 0x93, 0x79, 0xc9, 0x7d, - 0x52, 0x06, 0x20, 0x6b, 0xb2, 0x31, 0x93, 0x18, 0x32, 0x0b, 0xcc, 0x94, 0xb9, 0xed, 0xc2, 0xa0, - 0x30, 0xc2, 0x9e, 0xc0, 0x29, 0xeb, 0xd6, 0x4b, 0x45, 0x63, 0x23, 0x0f, 0x18, 0xd4, 0xbb, 0x05, - 0x53, 0xd1, 0x24, 0x0e, 0xd2, 0x80, 0x0d, 0x98, 0xec, 0xf0, 0xe7, 0x6e, 0x49, 0x7e, 0xed, 0x96, - 0xe4, 0xf7, 0x6e, 0x49, 0xbe, 0xfe, 0x59, 0xfe, 0x77, 0x33, 0x31, 0xff, 0x34, 0xcf, 0xff, 0x06, - 0x00, 0x00, 0xff, 0xff, 0x71, 0xd9, 0x96, 0xf1, 0x79, 0x04, 0x00, 0x00, + // 570 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0xcb, 0x6e, 0xd3, 0x40, + 0x14, 0x65, 0x62, 0xe7, 0x75, 0x93, 0x56, 0x61, 0x04, 0xc8, 0x42, 0x28, 0xb2, 0x2c, 0x16, 0xde, + 0x90, 0x4a, 0xe5, 0x03, 0x10, 0x6e, 0x5a, 0xc9, 0x42, 0x54, 0xed, 0xb4, 0x62, 0xc7, 0xc2, 0x2d, + 0x43, 0xb1, 0xe4, 0x17, 0x33, 0xe3, 0x45, 0x96, 0x2c, 0xf8, 0x07, 0xc4, 0x17, 0xc0, 0x9f, 0xb0, + 0xe4, 0x13, 0x50, 0xf8, 0x11, 0x74, 0xe7, 0x61, 0x7b, 0x55, 0xb1, 0x9b, 0x73, 0x4e, 0xee, 0xf8, + 0x9e, 0x39, 0xf7, 0x06, 0x96, 0x4d, 0x7b, 0x53, 0xe4, 0xb7, 0x9b, 0x46, 0xd4, 0xaa, 0xa6, 0xb3, + 0xbc, 0x52, 0x5c, 0x54, 0x59, 0x11, 0x25, 0x30, 0x49, 0x72, 0x55, 0x66, 0x0d, 0xa5, 0xe0, 0x27, + 0xb9, 0x92, 0x01, 0x09, 0xbd, 0xd8, 0x67, 0xfa, 0x4c, 0x9f, 0xc3, 0xf8, 0xb5, 0x52, 0x42, 0x06, + 0xa3, 0xd0, 0x8b, 0x17, 0xc7, 0x87, 0x1b, 0x57, 0xb7, 0x41, 0x9a, 0x19, 0x31, 0xda, 0x80, 0x7f, + 0x91, 0xe5, 0x82, 0xae, 0xc0, 0x7b, 0xc3, 0x77, 0x01, 0x09, 0x49, 0xec, 0x33, 0x3c, 0xd2, 0x47, + 0x30, 0x3e, 0xa9, 0xdb, 0x4a, 0x05, 0x23, 0xcd, 0x19, 0x10, 0xbd, 0x07, 0x2f, 0xc9, 0x15, 0x7d, + 0x0a, 0x33, 0xf3, 0xe9, 0x74, 0x6b, 0x6b, 0x3a, 0x4c, 0x9f, 0xc1, 0xfc, 0x42, 0xd4, 0x1f, 0xf3, + 0x82, 0xa7, 0x5b, 0x5b, 0xdc, 0x13, 0xa8, 0x5e, 0xe7, 0x25, 0x97, 0x2a, 0x2b, 0x9b, 0xc0, 0x0b, + 0x49, 0xec, 0xb1, 0x9e, 0x88, 0x5e, 0xc1, 0xd4, 0xfe, 0x94, 0x1e, 0xc2, 0xa8, 0xbb, 0x7c, 0x94, + 0x6e, 0xff, 0xd3, 0xcf, 0x0f, 0x02, 0x3e, 0x9e, 0x86, 0x86, 0xe6, 0xc6, 0x10, 0x05, 0xff, 0x7a, + 0xd7, 0x70, 0xdb, 0x92, 0x3e, 0xd3, 0x10, 0x16, 0x57, 0x4a, 0xe4, 0xd5, 0xdd, 0xbb, 0xac, 0x68, + 0xb9, 0xee, 0x67, 0xce, 0x86, 0x14, 0x3a, 0x4d, 0x2b, 0x65, 0x64, 0x5f, 0xb7, 0xdb, 0x61, 0xf4, + 0x92, 0xd4, 0x75, 0x61, 0xc4, 0x71, 0x48, 0xe2, 0x19, 0xeb, 0x09, 0xba, 0x06, 0x38, 0x2b, 0xea, + 0xcc, 0xd6, 0x4e, 0x42, 0x12, 0x13, 0x36, 0x60, 0xa2, 0x23, 0x98, 0x62, 0xa7, 0x6f, 0xb3, 0xa6, + 0xf7, 0x46, 0xee, 0xf3, 0xf6, 0x9d, 0xc0, 0xf2, 0xb2, 0xe5, 0x62, 0xc7, 0xf8, 0xe7, 0x96, 0x4b, + 0x85, 0x4f, 0xb4, 0x4d, 0xac, 0xc5, 0xd1, 0x36, 0xc1, 0xc8, 0xb4, 0xae, 0x2d, 0xce, 0x99, 0x01, + 0xf4, 0x09, 0x4c, 0xae, 0x8a, 0xfc, 0x96, 0xcb, 0xc0, 0xd3, 0xe3, 0x61, 0x11, 0x3a, 0xb3, 0x6f, + 0x2d, 0xb5, 0xb3, 0x19, 0xeb, 0x30, 0x0d, 0x60, 0x7a, 0xd9, 0x66, 0x95, 0x6a, 0x4b, 0xed, 0x6b, + 0xce, 0x1c, 0xc4, 0xdb, 0x18, 0x2f, 0x6b, 0x65, 0x1c, 0xcd, 0x98, 0x45, 0xd1, 0x17, 0x02, 0x07, + 0xb6, 0x39, 0xd9, 0xd4, 0x95, 0xe4, 0x98, 0xc0, 0xa9, 0x10, 0x2e, 0x81, 0x53, 0x21, 0xe8, 0x11, + 0x4c, 0x19, 0x97, 0x6d, 0xa1, 0x5c, 0x88, 0x8f, 0x7b, 0xa3, 0xae, 0xb6, 0x2d, 0x14, 0x73, 0xbf, + 0xa2, 0x2f, 0x06, 0x2d, 0x7a, 0xba, 0xe2, 0x61, 0x5f, 0x61, 0x95, 0xbe, 0xeb, 0xe8, 0x2b, 0x81, + 0xc5, 0xe0, 0x1e, 0x1a, 0xbb, 0x05, 0xd1, 0x4d, 0x2c, 0x8e, 0x57, 0x7d, 0xb1, 0xe1, 0x99, 0x5b, + 0xa0, 0x25, 0x90, 0x73, 0x3b, 0x18, 0xe4, 0x1c, 0xe3, 0xc0, 0xa5, 0x70, 0xdf, 0x1c, 0xc4, 0x81, + 0x34, 0x33, 0x22, 0xbe, 0xd1, 0xc9, 0xa7, 0xac, 0xba, 0xe3, 0x1f, 0xec, 0xf3, 0x39, 0x18, 0xfd, + 0x24, 0x70, 0x90, 0x96, 0x4d, 0x2d, 0xd4, 0x3d, 0x49, 0x9d, 0x89, 0xac, 0xe4, 0x2e, 0x29, 0x0d, + 0x90, 0xd5, 0xd9, 0xe8, 0x39, 0xf4, 0x99, 0x01, 0x7a, 0xca, 0xec, 0x6e, 0x61, 0x50, 0x18, 0x61, + 0x4f, 0xe0, 0x94, 0x75, 0xcb, 0x25, 0x83, 0xb1, 0x96, 0x07, 0x0c, 0xea, 0xdd, 0x7a, 0xc9, 0x60, + 0x12, 0x7a, 0xb1, 0xc7, 0x06, 0x4c, 0xb2, 0xfa, 0xb5, 0x5f, 0x93, 0xdf, 0xfb, 0x35, 0xf9, 0xb3, + 0x5f, 0x93, 0x6f, 0x7f, 0xd7, 0x0f, 0x6e, 0x26, 0xfa, 0x7f, 0xe6, 0xe5, 0xbf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x36, 0x47, 0xff, 0x96, 0x77, 0x04, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index ea8c71477..b62dcc13f 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -27,7 +27,7 @@ message Attr { string Key = 1; uint64 Type = 2; string StringValue = 3; - uint64 UintValue = 4; + int64 IntValue = 4; bool BoolValue = 5; double FloatValue = 6; } diff --git a/pql/ast.go b/pql/ast.go index 3168e5ec0..57cdf044d 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -30,6 +30,44 @@ type Call struct { Children []*Call } +// UintArg is for reading the value at key from call.Args as a uint64. The value +// is assumed to be an int64 and then cast to a uint64. An error is returned if +// the value is not found, or not an int64. +func (c *Call) UintArg(key string) (uint64, bool, error) { + val, ok := c.Args[key] + if !ok { + return 0, false, nil + } + switch tval := val.(type) { + case int64: + return uint64(tval), true, nil + case uint64: + return tval, true, nil + default: + return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Calll.UintArg", tval, tval) + } +} + +func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { + val, ok := c.Args[key] + if !ok { + return nil, false, nil + } + + switch tval := val.(type) { + case []uint64: + return tval, true, nil + case []int64: + ret := make([]uint64, len(tval)) + for i, v := range tval { + ret[i] = uint64(v) + } + return ret, true, nil + default: + return nil, true, fmt.Errorf("unexpected type %T in UintSliceArg, val %v", tval, tval) + } +} + // Keys returns a list of argument keys in sorted order. func (c *Call) Keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/pql/parser.go b/pql/parser.go index 346a47ccb..f7d93ab81 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -169,7 +169,7 @@ func (p *Parser) parseArgs() (map[string]interface{}, error) { case STRING: value = lit case INTEGER: - v, err := strconv.ParseUint(lit, 10, 64) + v, err := strconv.ParseInt(lit, 10, 64) if err != nil { return nil, err } @@ -226,7 +226,7 @@ func (p *Parser) parseList() ([]interface{}, error) { case STRING: values = append(values, lit) case INTEGER: - v, err := strconv.ParseUint(lit, 10, 64) + v, err := strconv.ParseInt(lit, 10, 64) if err != nil { return nil, err } diff --git a/pql/parser_test.go b/pql/parser_test.go index e5bda9ca9..66b84f6be 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -50,7 +50,7 @@ func TestParser_Parse(t *testing.T) { &pql.Call{ Name: "Count", Children: []*pql.Call{ - {Name: "Bitmap", Args: map[string]interface{}{"id": uint64(100)}}, + {Name: "Bitmap", Args: map[string]interface{}{"id": int64(100)}}, }, }, ) { @@ -69,7 +69,7 @@ func TestParser_Parse(t *testing.T) { Args: map[string]interface{}{ "key": "value", "foo": "bar", - "age": uint64(12), + "age": int64(12), "bool0": true, "bool1": false, "x": nil, @@ -100,6 +100,24 @@ func TestParser_Parse(t *testing.T) { } }) + // Parse with float arguments. + t.Run("WithNegativeArgs", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": -12.25, + "foo": int64(-13), + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + // Parse with both child calls and arguments. t.Run("ChildrenAndArguments", func(t *testing.T) { q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`) @@ -110,9 +128,9 @@ func TestParser_Parse(t *testing.T) { Name: "TopN", Children: []*pql.Call{{ Name: "Bitmap", - Args: map[string]interface{}{"id": uint64(100), "frame": "other"}, + Args: map[string]interface{}{"id": int64(100), "frame": "other"}, }}, - Args: map[string]interface{}{"n": uint64(3), "frame": "f"}, + Args: map[string]interface{}{"n": int64(3), "frame": "f"}, }, ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) @@ -129,7 +147,7 @@ func TestParser_Parse(t *testing.T) { Name: "TopN", Args: map[string]interface{}{ "frame": "f", - "ids": []interface{}{uint64(0), uint64(10), uint64(30)}, + "ids": []interface{}{int64(0), int64(10), int64(30)}, }, }, ) { diff --git a/pql/scanner.go b/pql/scanner.go index e51f0ab52..a0146725a 100644 --- a/pql/scanner.go +++ b/pql/scanner.go @@ -34,7 +34,7 @@ func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { } else if isIdentFirstChar(ch) { s.unread() return s.scanIdent() - } else if isDigit(ch) { + } else if isDigit(ch) || ch == '-' { s.unread() return s.scanNumber() } else if ch == '"' || ch == '\'' { @@ -140,16 +140,17 @@ func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) { return IDENT, pos, lit } -// scanNumber consumes consecutive digits and up to one '.' character. +// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character. func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { pos = s.pos tok = INTEGER var buf bytes.Buffer var seenDot bool + first := true for { ch := s.read() - if !isDigit(ch) && (seenDot || ch != '.') { + if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') { s.unread() break } @@ -158,6 +159,7 @@ func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { tok = FLOAT } buf.WriteRune(ch) + first = false } return tok, pos, buf.String() } diff --git a/server/server_test.go b/server/server_test.go index 0c717c16d..64bfa2d42 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -115,6 +115,8 @@ func TestMain_SetBitmapAttrs(t *testing.T) { t.Fatal(err) } else if err := client.CreateFrame(context.Background(), "d", "z", pilosa.FrameOptions{}); err != nil { t.Fatal(err) + } else if err := client.CreateFrame(context.Background(), "d", "neg", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) } // Set bits on different bitmaps in different frames. @@ -124,15 +126,19 @@ func TestMain_SetBitmapAttrs(t *testing.T) { t.Fatal(err) } else if _, err := m.Query("db=d", `SetBit(id=2, frame="z", profileID=100)`); err != nil { t.Fatal(err) + } else if _, err := m.Query("db=d", `SetBit(id=3, frame="neg", profileID=100)`); err != nil { + t.Fatal(err) } // Set bitmap attributes. if _, err := m.Query("db=d", `SetBitmapAttrs(id=1, frame="x.n", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("db=d", `SetBitmapAttrs(id=2, frame="x.n", x=200)`); err != nil { + } else if _, err := m.Query("db=d", `SetBitmapAttrs(id=2, frame="x.n", x=-200)`); err != nil { t.Fatal(err) } else if _, err := m.Query("db=d", `SetBitmapAttrs(id=2, frame="z", x=300)`); err != nil { t.Fatal(err) + } else if _, err := m.Query("db=d", `SetBitmapAttrs(id=3, frame="neg", x=-0.44)`); err != nil { + t.Fatal(err) } // Query bitmap x.n/1. @@ -145,7 +151,7 @@ func TestMain_SetBitmapAttrs(t *testing.T) { // Query bitmap x.n/2. if res, err := m.Query("db=d", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":200},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -153,12 +159,24 @@ func TestMain_SetBitmapAttrs(t *testing.T) { t.Fatal(err) } - // Query bitmap after reopening. + // Query bitmaps after reopening. if res, err := m.Query("db=d&profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } + + if res, err := m.Query("db=d&profiles=true", `Bitmap(id=3, frame="neg")`); err != nil { + t.Fatal(err) + } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { + t.Fatalf("unexpected result(reopen): %s", res) + } + // Query bitmap x.n/2. + if res, err := m.Query("db=d", `Bitmap(id=2, frame="x.n")`); err != nil { + t.Fatal(err) + } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { + t.Fatalf("unexpected result: %s", res) + } } // Ensure program can set profile attributes and retrieve them.