scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 11:27:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into wip-api-refactor

Browse source
This commit is contained in:
Matt Jaffee 2018-06-22 12:46:19 -05:00
commit 7a5adf7428
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
25 changed files with 4089 additions and 964 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-peg test
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@ -92,8 +92,11 @@ generate-protoc: require-protoc require-protoc-gen-gofast
generate-stringer:
go generate github.com/pilosa/pilosa
generate-pql: require-peg
cd pql && peg -inline pql.peg && cd ..
# `go generate` all needed packages
generate: generate-protoc generate-stringer
generate: generate-protoc generate-stringer generate-pql
# Create Docker image from Dockerfile
docker:
@ -128,7 +131,10 @@ require-protoc-gen-gofast:
require-protoc:
$(call require,protoc)
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer
require-peg:
$(call require,peg)
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg
install-dep:
go get -u github.com/golang/dep/cmd/dep
@ -141,3 +147,6 @@ install-protoc-gen-gofast:
install-protoc:
@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
install-peg:
go get github.com/pointlander/peg

1
api.go
View file

@ -685,7 +685,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
if err != nil {
return errors.Wrap(err, "getting field")
}
// Import into fragment.
err = field.ImportValue(req.ColumnIDs, req.Values)
if err != nil {

2
cluster.go
View file

@ -229,7 +229,7 @@ type Cluster struct {
// Threshold for logging long-running queries
LongQueryTime time.Duration
// Maximum number of SetBit() or ClearBit() commands per request.
// Maximum number of Set() or Clear() commands per request.
MaxWritesPerRequest int
// EventReceiver receives NodeEvents pertaining to node membership.

52
ctl/import_test.go
View file

@ -175,3 +175,55 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}
func TestImportCommand_BugOverwriteValue(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import-value.csv")
file.Write([]byte("0,17\n"))
ctx := context.Background()
if err != nil {
t.Fatal(err)
}
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
cm.Index = "i"
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
file.Close()
file, err = ioutil.TempFile("", "import-value2.csv")
file.Write([]byte("0,16\n"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
file.Close()
file, err = ioutil.TempFile("", "import-value3.csv")
file.Write([]byte("0,19\n"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
}

6
enterprise/b/containers_btree.go
View file

@ -160,6 +160,12 @@ func (btc *BTreeContainers) Size() int {
return btc.tree.Len()
}
func (btc *BTreeContainers) Reset() {
btc.tree = TreeNew(cmp)
btc.lastKey = 0
btc.lastContainer = nil
}
func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
e, ok := btc.tree.Seek(key)
if ok {

157
executor.go
View file

@ -48,7 +48,7 @@ type Executor struct {
// Client used for remote requests.
client InternalQueryClient
// Maximum number of SetBit() or ClearBit() commands per request.
// Maximum number of Set() or Clear() commands per request.
MaxWritesPerRequest int
// Stores key/id translation data.
@ -178,12 +178,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
case "Max":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeMax(ctx, index, c, slices, opt)
case "ClearBit":
case "Clear":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeCount(ctx, index, c, slices, opt)
case "SetBit":
case "Set":
return e.executeSetBit(ctx, index, c, opt)
case "SetValue":
return nil, e.executeSetValue(ctx, index, c, opt)
@ -340,17 +340,17 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
return nil, err
}
// Attach attributes for Bitmap() calls.
// Attach attributes for Row() calls.
// If the column label is used then return column attributes.
// If the row label is used then return bitmap attributes.
row, _ := other.(*Row)
if c.Name == "Bitmap" {
if c.Name == "Row" {
if opt.ExcludeRowAttrs {
row.Attrs = map[string]interface{}{}
} else {
idx := e.Holder.Index(index)
if idx != nil {
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
if columnID, ok, err := c.UintArg("_" + columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
return nil, errors.Wrap(err, "getting column attrs")
@ -359,9 +359,10 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
} else if err != nil {
return nil, err
} else {
field, _ := c.Args["field"].(string)
if fr := idx.Field(field); fr != nil {
rowID, _, err := c.UintArg(rowLabel)
// field, _ := c.Args["field"].(string)
fieldName, _ := c.FieldArg()
if fr := idx.Field(fieldName); fr != nil {
rowID, _, err := c.UintArg(fieldName)
if err != nil {
return nil, errors.Wrap(err, "getting row")
}
@ -386,7 +387,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
// executeBitmapCallSlice executes a bitmap call for a single slice.
func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
switch c.Name {
case "Bitmap":
case "Row":
return e.executeBitmapSlice(ctx, index, c, slice)
case "Difference":
return e.executeDifferenceSlice(ctx, index, c, slice)
@ -585,7 +586,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
field, _ := c.Args["field"].(string)
field, _ := c.Args["_field"].(string)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -675,24 +676,24 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
}
// Fetch field & row label based on argument.
field, _ := c.Args["field"].(string)
if field == "" {
field = defaultField
fieldName, err := c.FieldArg()
if err != nil {
return nil, errors.New("Row() argument required: field")
}
f := e.Holder.Field(index, field)
f := e.Holder.Field(index, fieldName)
if f == nil {
return nil, ErrFieldNotFound
}
rowID, rowOK, rowErr := c.UintArg(rowLabel)
rowID, rowOK, rowErr := c.UintArg(fieldName)
if rowErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr)
return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr)
}
if !rowOK {
return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel)
return nil, fmt.Errorf("Row() must specify %v", rowLabel)
}
frag := e.Holder.Fragment(index, field, ViewStandard, slice)
frag := e.Holder.Fragment(index, fieldName, ViewStandard, slice)
if frag == nil {
return NewRow(), nil
}
@ -728,10 +729,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
return e.executeBSIGroupRangeSlice(ctx, index, c, slice)
}
// Parse field, use default if unset.
field, _ := c.Args["field"].(string)
if field == "" {
field = defaultField
// Parse field.
fieldName, err := c.FieldArg()
if err != nil {
return nil, errors.New("Range() argument required: field")
}
// Retrieve column label.
@ -741,13 +742,13 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Retrieve base field.
f := idx.Field(field)
f := idx.Field(fieldName)
if f == nil {
return nil, ErrFieldNotFound
}
// Read row & column id.
rowID, rowOK, err := c.UintArg(rowLabel)
rowID, rowOK, err := c.UintArg(fieldName)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err)
}
@ -756,7 +757,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Parse start time.
startTimeStr, ok := c.Args["start"].(string)
startTimeStr, ok := c.Args["_start"].(string)
if !ok {
return nil, errors.New("Range() start time required")
}
@ -766,7 +767,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Parse end time.
endTimeStr, ok := c.Args["end"].(string)
endTimeStr, ok := c.Args["_end"].(string)
if !ok {
return nil, errors.New("Range() end time required")
}
@ -784,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based views.
row := &Row{}
for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, field, view, slice)
f := e.Holder.Fragment(index, fieldName, view, slice)
if f == nil {
continue
}
@ -994,11 +995,11 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
return n, nil
}
// executeClearBit executes a ClearBit() call.
// executeClearBit executes a Clear() call.
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
field, ok := c.Args["field"].(string)
if !ok {
return false, errors.New("ClearBit() field required")
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("Clear() argument required: field")
}
// Retrieve field.
@ -1006,30 +1007,30 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
if idx == nil {
return false, ErrIndexNotFound
}
f := idx.Field(field)
f := idx.Field(fieldName)
if f == nil {
return false, ErrFieldNotFound
}
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading ClearBit() row: %v", err)
return false, fmt.Errorf("reading Clear() row: %v", err)
} else if !ok {
return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel)
return false, fmt.Errorf("Clear() row argument '%v' required", rowLabel)
}
colID, ok, err := c.UintArg(columnLabel)
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
return false, fmt.Errorf("reading ClearBit() column: %v", err)
return false, fmt.Errorf("reading Clear() column: %v", err)
} else if !ok {
return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel)
return false, fmt.Errorf("Clear() col argument '%v' required", columnLabel)
}
return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt)
}
// executeClearBitField executes a ClearBit() call for a single view.
// executeClearBitField executes a Clear() call for a single view.
func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
@ -1059,11 +1060,11 @@ func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pq
return ret, nil
}
// executeSetBit executes a SetBit() call.
// executeSetBit executes a Set() call.
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
field, ok := c.Args["field"].(string)
if !ok {
return false, errors.New("SetBit() field required: field")
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("Set() argument required: field")
}
// Retrieve field.
@ -1071,28 +1072,28 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
if idx == nil {
return false, ErrIndexNotFound
}
f := idx.Field(field)
f := idx.Field(fieldName)
if f == nil {
return false, ErrFieldNotFound
}
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading SetBit() row: %v", err)
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel)
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
colID, ok, err := c.UintArg(columnLabel)
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
return false, fmt.Errorf("reading SetBit() column: %v", err)
return false, fmt.Errorf("reading Set() column: %v", err)
} else if !ok {
return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel)
return false, fmt.Errorf("Set() column argument '%v' required", columnLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["timestamp"].(string)
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
@ -1104,7 +1105,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
// executeSetBitField executes a SetBit() call for a specific view.
// executeSetBitField executes a Set() call for a specific view.
func (e *Executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
@ -1198,7 +1199,7 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal
// executeSetRowAttrs executes a SetRowAttrs() call.
func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
fieldName, ok := c.Args["field"].(string)
fieldName, ok := c.Args["_field"].(string)
if !ok {
return errors.New("SetRowAttrs() field required")
}
@ -1210,7 +1211,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
}
// Parse labels.
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg("_" + rowLabel)
if err != nil {
return fmt.Errorf("reading SetRowAttrs() row: %v", err)
} else if !ok {
@ -1219,8 +1220,8 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "field")
delete(attrs, rowLabel)
delete(attrs, "_field")
delete(attrs, "_"+rowLabel)
// Set attributes.
if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
@ -1258,7 +1259,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
// Collect attributes by field/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, c := range calls {
field, ok := c.Args["field"].(string)
field, ok := c.Args["_field"].(string)
if !ok {
return nil, errors.New("SetRowAttrs() field required")
}
@ -1269,7 +1270,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
return nil, ErrFieldNotFound
}
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg("_" + rowLabel)
if err != nil {
return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel)
} else if !ok {
@ -1278,8 +1279,8 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "field")
delete(attrs, rowLabel)
delete(attrs, "_field")
delete(attrs, "_"+rowLabel)
// Create field group, if not exists.
fieldMap := m[field]
@ -1348,14 +1349,14 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
return ErrIndexNotFound
}
col, okCol, errCol := c.UintArg(columnLabel)
col, okCol, errCol := c.UintArg("_" + columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, columnLabel)
delete(attrs, "_"+columnLabel)
delete(attrs, "field")
// Set attributes.
@ -1420,9 +1421,9 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
v, err = pb.Results[i].N, nil
case "SetBit":
case "Set":
v, err = pb.Results[i].Changed, nil
case "ClearBit":
case "Clear":
v, err = pb.Results[i].Changed, nil
case "SetRowAttrs":
case "SetColumnAttrs":
@ -1493,6 +1494,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
case resp := <-ch:
// On error retry against remaining nodes. If an error returns then
// the context will cancel and cause all open goroutines to return.
if resp.err != nil {
// Filter out unavailable nodes.
nodes = Nodes(nodes).Filter(resp.node)
@ -1591,27 +1593,38 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu
}
func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error {
var colKey, rowKey, fieldName string
if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" {
// Positional args in new PQL syntax require special handling here.
colKey = "_" + columnLabel
fieldName, _ = c.FieldArg()
rowKey = fieldName
} else {
colKey = "col"
fieldName = callArgString(c, "field")
rowKey = "row"
}
// Translate column key.
if idx.Keys() {
if value := callArgString(c, "col"); value != "" {
if value := callArgString(c, colKey); value != "" {
ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value})
if err != nil {
return err
}
c.Args["col"] = ids[0]
c.Args[colKey] = ids[0]
}
}
// Translate row key, if field is specified & key exists.
if fieldName := callArgString(c, "field"); fieldName != "" {
if fieldName != "" {
field := idx.Field(fieldName)
if field.Keys() {
if value := callArgString(c, "row"); value != "" {
if value := callArgString(c, rowKey); value != "" {
ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value})
if err != nil {
return err
}
c.Args["row"] = ids[0]
c.Args[rowKey] = ids[0]
}
}
}
@ -1644,7 +1657,7 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res
}
case []Pair:
if fieldName := callArgString(call, "field"); fieldName != "" {
if fieldName := callArgString(call, "_field"); fieldName != "" {
field := idx.Field(fieldName)
if field.Keys() {
other := make([]Pair, len(result))
@ -1713,7 +1726,7 @@ func needsSlices(calls []*pql.Call) bool {
}
for _, call := range calls {
switch call.Name {
case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs":
case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs":
continue
case "Count", "TopN":
return true

313
executor_test.go
View file

@ -44,9 +44,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -54,7 +54,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
@ -63,7 +63,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit column attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -72,7 +72,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit row attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -93,9 +93,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -116,15 +116,15 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
`SetBit(field=f, row="bar", col="foo")`+"\n"+
`SetBit(field=f, row="baz", col="foo")`+"\n"+
`SetBit(field=f, row="bar", col="bat")`+"\n"+
`SetBit(field=f, row="bbb", col="aaa")`+"\n",
`Set("foo", f="bar")`+"\n"+
`Set("foo", f="baz")`+"\n"+
`Set("bat", f="bar")`+"\n"+
`Set("aaa", f="bbb")`+"\n",
), nil, nil); err != nil {
t.Fatal(err)
}
if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil {
if results, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f="bar")`), nil, nil); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(results, []interface{}{
&pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}},
@ -145,7 +145,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
hldr.SetBit("i", "general", 11, 4)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -177,7 +177,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -207,7 +207,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -240,7 +240,7 @@ func TestExecutor_Execute_Xor(t *testing.T) {
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -256,7 +256,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr.SetBit("i", "f", 10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
@ -264,7 +264,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
}
// Ensure a set query can be executed.
func TestExecutor_Execute_SetBit(t *testing.T) {
func TestExecutor_Execute_Set(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -276,7 +276,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Set(1, f=11)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
@ -287,7 +287,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := hldr.Row("i", "f", 11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Set(1, f=11)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
@ -296,6 +296,21 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
}
}
// Ensure old PQL syntax doesn't break anything too badly.
func TestExecutor_Execute_OldPQL(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// set a bit so the view gets created.
hldr.SetBit("i", "f", 1, 0)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(frame=f, row=11, col=1)`), nil, nil); err == nil || err.Error() != "unknown call: SetBit" {
t.Fatal("Expected error: 'unknown call: SetBit'")
}
}
// Ensure a SetValue() query can be executed.
func TestExecutor_Execute_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
@ -391,16 +406,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Set two attrs on f/10.
// Also set attrs on other bitmaps and fields to test isolation.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, field=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 200, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(xxx, 10, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -427,15 +442,15 @@ func TestExecutor_Execute_TopN(t *testing.T) {
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=1)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(field=f, row=10, col=0)
SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=other, row=0, col=0)
Set(0, f=0)
Set(1, f=0)
Set(`+strconv.Itoa(SliceWidth)+`, f=0)
Set(`+strconv.Itoa(SliceWidth+2)+`, f=0)
Set(`+strconv.Itoa((5*SliceWidth)+100)+`, f=0)
Set(0, f=10)
Set(`+strconv.Itoa(SliceWidth)+`, f=10)
Set(`+strconv.Itoa(SliceWidth)+`, f=20)
Set(0, other=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -444,7 +459,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: 0, Count: 5},
@ -467,22 +482,22 @@ func TestExecutor_Execute_TopN(t *testing.T) {
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row="foo", col="a")
SetBit(field=f, row="foo", col="b")
SetBit(field=f, row="foo", col="c")
SetBit(field=f, row="foo", col="d")
SetBit(field=f, row="foo", col="e")
SetBit(field=f, row="bar", col="a")
SetBit(field=f, row="bar", col="b")
SetBit(field=f, row="baz", col="b")
SetBit(field=other, row="foo", col="a")
Set("a", f="foo")
Set("b", f="foo")
Set("c", f="foo")
Set("d", f="foo")
Set("e", f="foo")
Set("a", f="bar")
Set("b", f="bar")
Set("b", f="baz")
Set("a", other="foo")
`), nil, nil); err != nil {
t.Fatal(err)
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(result, []interface{}{
[]pilosa.Pair{
@ -509,7 +524,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 4},
@ -543,7 +558,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -578,7 +593,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(other=100), n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
@ -602,7 +617,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -625,7 +640,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -658,20 +673,20 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=x, row=0, col=0)
SetBit(field=x, row=0, col=3)
SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(field=x, row=1, col=1)
SetBit(field=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`)
Set(0, x=0)
Set(3, x=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, x=0)
Set(1, x=1)
Set(`+strconv.Itoa(SliceWidth+2)+`, x=2)
SetValue(f=20, col=0)
SetValue(f=-5, col=1)
SetValue(f=-5, col=2)
SetValue(f=10, col=3)
SetValue(f=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(col=0, f=20)
SetValue(col=1, f=-5)
SetValue(col=2, f=-5)
SetValue(col=3, f=10)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, f=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, f=40)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, f=50)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, f=60)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -683,9 +698,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
cnt int64
}{
{filter: ``, exp: -5, cnt: 2},
{filter: `Bitmap(field=x, row=0)`, exp: 10, cnt: 1},
{filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1},
{filter: `Row(x=0)`, exp: 10, cnt: 1},
{filter: `Row(x=1)`, exp: -5, cnt: 1},
{filter: `Row(x=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
@ -709,9 +724,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
cnt int64
}{
{filter: ``, exp: 60, cnt: 1},
{filter: `Bitmap(field=x, row=0)`, exp: 60, cnt: 1},
{filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1},
{filter: `Row(x=0)`, exp: 60, cnt: 1},
{filter: `Row(x=1)`, exp: -5, cnt: 1},
{filter: `Row(x=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
@ -769,16 +784,16 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=x, row=0, col=0)
SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
Set(0, x=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, x=0)
SetValue(foo=20, col=0)
SetValue(bar=2000, col=0)
SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(other=1000, col=0)
SetValue(col=0, foo=20)
SetValue(col=0, bar=2000)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=40)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=50)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60)
SetValue(col=0, other=1000)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -792,7 +807,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(field=x, row=0), field=foo)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Row(x=0), field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -801,7 +816,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
// Ensure a range query can be executed.
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
func TestExecutor_Execute_Range(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
@ -818,23 +833,24 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
}
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(field=f, row=1, col=3, timestamp="2000-01-01T00:00")
SetBit(field=f, row=1, col=4, timestamp="2000-01-02T00:00")
SetBit(field=f, row=1, col=5, timestamp="2000-02-01T00:00")
SetBit(field=f, row=1, col=6, timestamp="2001-01-01T00:00")
SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00")
cc := test.MustParse(`
Set(2, f=1, 1999-12-31T00:00)
Set(3, f=1, 2000-01-01T00:00)
Set(4, f=1, 2000-01-02T00:00)
Set(5, f=1, 2000-02-01T00:00)
Set(6, f=1, 2001-01-01T00:00)
Set(7, f=1, 2002-01-01T02:00)
SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00")
SetBit(field=f, row=1, col=2, timestamp="2002-02-01T00:00")
SetBit(field=f, row=10, col=2, timestamp="2001-01-01T00:00")
`), nil, nil); err != nil {
Set(2, f=1, 1999-12-30T00:00)
Set(2, f=1, 2002-02-01T00:00)
Set(2, f=10, 2001-01-01T00:00)
`)
if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -843,7 +859,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
}
// Ensure a Range(bsiGroup) query can be executed.
func TestExecutor_Execute_Range(t *testing.T) {
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
@ -890,18 +906,18 @@ func TestExecutor_Execute_Range(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
Set(0, f=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, f=0)
SetValue(foo=20, col=50)
SetValue(bar=2000, col=50)
SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(other=1000, col=0)
SetValue(edge=100, col=0)
SetValue(edge=-100, col=1)
SetValue(col=50, foo=20)
SetValue(col=50, bar=2000)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=10)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=20)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60)
SetValue(col=0, other=1000)
SetValue(col=0, edge=100)
SetValue(col=1, edge=-100)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -969,7 +985,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
})
t.Run("BETWEEN", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [1, 1000])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(0 < other < 1000)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -978,7 +994,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Ensure that the NotNull code path gets run.
t.Run("NotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [0, 1000])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(-1 < other < 1000)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -1044,7 +1060,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Bitmap(field="f", row=10)` {
} else if query.String() != `Row(f=10)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{1}) {
t.Fatalf("unexpected slices: %+v", slices)
@ -1067,7 +1083,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
hldr.SetBit("i", "f", 10, SliceWidth+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -1104,7 +1120,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
hldr.SetBit("i", "f", 10, (2*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
@ -1134,7 +1150,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(col=2, field="f", row=10)` {
} else if query.String() != `Set(_col=2, f=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1152,7 +1168,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2)`), nil, nil); err != nil {
cc := test.MustParse("Set(2, f=10)")
if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil {
t.Fatal(err)
}
@ -1188,7 +1205,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(col=2, field="f", row=10, timestamp="2016-12-11T10:09")` {
} else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1208,7 +1225,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
cc := test.MustParse(`Set(2, f=10, 2016-12-11T10:09)`)
if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil {
t.Fatal(err)
}
@ -1251,11 +1269,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// slices and a second time to get the counts for a set of bitmaps.
switch remoteExecN {
case 0:
if query.String() != `TopN(field="f", n=3)` {
if query.String() != `TopN(_field="f", n=3)` {
t.Fatalf("unexpected query(0): %s", query.String())
}
case 1:
if query.String() != `TopN(field="f", ids=[0,10,30], n=3)` {
if query.String() != `TopN(_field="f", ids=[0,10,30], n=3)` {
t.Fatalf("unexpected query(1): %s", query.String())
}
default:
@ -1279,7 +1297,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
hldr.SetBit("i", "f", 30, (4*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -1290,6 +1308,55 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
}
}
// Ensure a remote query can set RowAttrs
func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) {
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` {
t.Fatalf("unexpected query: %s", query.String())
}
return []interface{}{}, nil
}
// Create local executor data.
// The local node owns slice 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
f := hldr.Field("i", "f")
s.Handler.API.Holder = hldr.Holder
hldr.SetBit("i", "f", 10, SliceWidth+1)
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
} else if m, err := f.RowAttrStore().Attrs(10); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"bat": true, "baz": int64(123)}) {
t.Fatalf("unexpected bitmap attr: %#v", m)
}
}
// Ensure executor returns an error if too many writes are in a single request.
func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
hldr := test.MustOpenHolder()
@ -1297,13 +1364,13 @@ func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.MaxWritesPerRequest = 3
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Set() Clear() Set() Set()`), nil, nil); err != pilosa.ErrTooManyWrites {
t.Fatalf("unexpected error: %s", err)
}
}
// Ensure SetColumnAttrs doesn't save `field` as an attribute
func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
@ -1314,11 +1381,11 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// SetColumnAttrs call should exclude the field attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil)
_, err := e.Execute(context.Background(), "i", test.MustParse("Set(10, f=1)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(field='f', col=10, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(10, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}
@ -1331,11 +1398,11 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
}
// SetColumnAttrs call should not break if field is not specified
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=20)"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("Set(20, f=10)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(20, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}

4
fragment.go
View file

@ -1873,11 +1873,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Only sync the standard block.
for j := 0; j < len(set.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.slice*SliceWidth)+set.columnIDs[j], f.field, set.rowIDs[j])
count++
}
for j := 0; j < len(clear.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.slice*SliceWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j])
count++
}

33
fragment_internal_test.go
View file

@ -213,6 +213,39 @@ func TestFragment_SetValue(t *testing.T) {
t.Fatal(err)
}
})
t.Run("Crash", func(t *testing.T) {
f := mustOpenFragment("i", "f", ViewStandard, 0, "")
defer f.Close()
// Set value.
if changed, err := f.setValue(0, 32, 17); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
if changed, err := f.setValue(0, 32, 16); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
if changed, err := f.setValue(0, 32, 19); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.value(0, 32); err != nil {
t.Fatal(err)
} else if value != 19 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected to exist")
}
})
}
// Ensure a fragment can sum values.

2
http/client_test.go
View file

@ -157,7 +157,7 @@ func TestClient_MultiNode(t *testing.T) {
topN := 4
queryRequest := &internal.QueryRequest{
Query: fmt.Sprintf(`TopN(field="%s", n=%d)`, "f", topN),
Query: fmt.Sprintf(`TopN(f, n=%d)`, topN),
Remote: false,
}
result, err := client[0].Query(context.Background(), "i", queryRequest)

194
pql/ast.go
View file

@ -26,6 +26,186 @@ import (
// Query represents a PQL query.
type Query struct {
Calls []*Call
lastField string
lastCond Token
inList bool
callStack []*Call
conditional []string
}
func (q *Query) startCall(name string) {
newCall := &Call{Name: name}
q.callStack = append(q.callStack, newCall)
if len(q.callStack) == 1 {
q.Calls = append(q.Calls, newCall)
} else {
calls := q.callStack[len(q.callStack)-2].Children
q.callStack[len(q.callStack)-2].Children = append(calls, newCall)
}
}
func (q *Query) endCall() {
q.callStack = q.callStack[:len(q.callStack)-1]
}
func (q *Query) addPosNum(key, value string) {
q.addField(key)
q.addNumVal(value)
}
func (q *Query) addPosStr(key, value string) {
q.addField(key)
q.addVal(value)
}
func (q *Query) startConditional() {
q.conditional = make([]string, 0)
call := q.callStack[len(q.callStack)-1]
if call.Args == nil {
call.Args = make(map[string]interface{})
}
}
func (q *Query) condAdd(val string) {
q.conditional = append(q.conditional, val)
}
func (q *Query) endConditional() {
// do stuff
if len(q.conditional) != 5 {
panic(fmt.Sprintf("conditional of wrong length: %#v", q.conditional))
}
low, _ := strconv.ParseInt(q.conditional[0], 10, 64)
field := q.conditional[2]
high, _ := strconv.ParseInt(q.conditional[4], 10, 64)
if q.conditional[1] == "<" {
low++
}
if q.conditional[3] == "<=" {
high++
}
call := q.callStack[len(q.callStack)-1]
call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}}
q.conditional = nil
}
func (q *Query) addField(field string) {
if q.lastField != "" {
panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField))
}
q.lastField = field
call := q.callStack[len(q.callStack)-1]
if call.Args == nil {
call.Args = make(map[string]interface{})
}
}
func (q *Query) addVal(val interface{}) {
if q.lastField == "" {
panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val))
}
call := q.callStack[len(q.callStack)-1]
if q.inList {
list := call.Args[q.lastField].([]interface{})
call.Args[q.lastField] = append(list, val)
return
}
if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: val,
}
} else {
call.Args[q.lastField] = val
}
q.lastField = ""
q.lastCond = ILLEGAL
}
func (q *Query) addNumVal(val string) {
if q.lastField == "" {
panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val))
}
var ival interface{}
var err error
if strings.Contains(val, ".") {
ival, err = strconv.ParseFloat(val, 64)
} else {
ival, err = strconv.ParseInt(val, 10, 64)
}
if err != nil {
panic(err)
}
call := q.callStack[len(q.callStack)-1]
if q.inList {
if q.lastCond != ILLEGAL {
list := call.Args[q.lastField].(*Condition).Value.([]interface{})
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: append(list, ival),
}
} else {
list := call.Args[q.lastField].([]interface{})
call.Args[q.lastField] = append(list, ival)
}
return
} else if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: ival,
}
} else {
call.Args[q.lastField] = ival
}
q.lastField = ""
q.lastCond = ILLEGAL
}
func (q *Query) startList() {
call := q.callStack[len(q.callStack)-1]
if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: make([]interface{}, 0),
}
} else {
call.Args[q.lastField] = make([]interface{}, 0)
}
q.inList = true
}
func (q *Query) endList() {
q.inList = false
q.lastField = ""
q.lastCond = ILLEGAL
}
func (q *Query) addGT() {
q.lastCond = GT
}
func (q *Query) addLT() {
q.lastCond = LT
}
func (q *Query) addGTE() {
q.lastCond = GTE
}
func (q *Query) addLTE() {
q.lastCond = LTE
}
func (q *Query) addEQ() {
q.lastCond = EQ
}
func (q *Query) addNEQ() {
q.lastCond = NEQ
}
func (q *Query) addBTWN() {
q.lastCond = BETWEEN
}
// WriteCallN returns the number of mutating calls.
@ -33,7 +213,7 @@ func (q *Query) WriteCallN() int {
var n int
for _, call := range q.Calls {
switch call.Name {
case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs":
case "Set", "Clear", "SetRowAttrs", "SetColumnAttrs":
n++
}
}
@ -73,6 +253,18 @@ type Call struct {
Children []*Call
}
// FieldArg determines which key-value pair contains the field and rowID,
// in the case of arguments like Set(colID, field=rowID).
// Returns the field as a string if present, or an error if not.
func (c *Call) FieldArg() (string, error) {
for arg := range c.Args {
if !strings.HasPrefix(arg, "_") {
return arg, nil
}
}
return "", fmt.Errorf("No field argument specified")
}
// UintArg is for reading the value at key from call.Args as a uint64. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a uint64 or an int64 and

299
pql/parser.go
View file

@ -15,10 +15,11 @@
package pql
import (
"fmt"
"io"
"strconv"
"io/ioutil"
"strings"
"github.com/pkg/errors"
)
// TimeFormat is the go-style time format used to parse string dates.
@ -26,13 +27,16 @@ const TimeFormat = "2006-01-02T15:04"
// Parser represents a parser for the PQL language.
type Parser struct {
scanner *bufScanner
r io.Reader
//scanner *bufScanner
PQL
}
// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
return &Parser{
scanner: newBufScanner(r),
r: r,
// scanner: newBufScanner(r),
}
}
@ -43,287 +47,18 @@ func ParseString(s string) (*Query, error) {
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
q := &Query{}
for {
call, err := p.parseCall()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
q.Calls = append(q.Calls, call)
}
// Require at least one call.
if len(q.Calls) == 0 {
return nil, io.ErrUnexpectedEOF
}
return q, nil
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (*Call, error) {
var c Call
// Read call name.
tok, pos, lit := p.scanIgnoreWhitespace()
if tok == EOF {
return nil, io.EOF
} else if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}
c.Name = lit
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse children first.
children, err := p.parseChildren()
buf, err := ioutil.ReadAll(p.r)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "reading buffer to parse")
}
c.Children = children
// If next token is a closing paren then exit.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
return &c, nil
} else if tok == IDENT {
p.unscan(1)
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit)
p.PQL = PQL{
Buffer: string(buf),
}
// Parse key/value arguments.
args, err := p.parseArgs()
p.Init()
err = p.PQL.Parse()
if err != nil {
return nil, err
}
c.Args = args
// Scan closing parenthesis.
if err := p.expect(RPAREN); err != nil {
return nil, err
}
return &c, nil
}
// parseChildren parses call children.
func (p *Parser) parseChildren() ([]*Call, error) {
var offset int
var children []*Call
for {
// Ensure next two tokens are IDENT+LPAREN.
if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT {
p.unscanIgnoreWhitespace(1 + offset)
return children, nil
}
if tok, _, _ := p.scan(); tok != LPAREN {
p.unscanIgnoreWhitespace(2 + offset)
return children, nil
}
// Push tokens back on scanner and parse as a call.
p.unscan(2)
child, err := p.parseCall()
if err != nil {
return nil, err
}
children = append(children, child)
// Exit if closing paren.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
p.unscan(1)
return children, nil
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
}
// Make sure comma is unscanned.
offset = 1
}
}
// parseArgs parses key/value arguments.
func (p *Parser) parseArgs() (map[string]interface{}, error) {
args := make(map[string]interface{})
for {
// Parse key.
tok, pos, lit := p.scanIgnoreWhitespace()
if tok == RPAREN {
p.unscan(1)
return args, nil
} else if tok != IDENT {
return nil, parseErrorf(pos, "expected argument key, found %q", lit)
}
key := lit
// Expect '=' or a comparison next.
var op Token
switch tok, pos, lit := p.scanIgnoreWhitespace(); tok {
case ASSIGN:
case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN:
op = tok
default:
return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit)
}
// Parse value.
var value interface{}
tok, pos, lit = p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
value = true
} else if lit == "false" {
value = false
} else if lit == "null" {
value = nil
} else {
value = lit
}
case STRING:
value = lit
case INTEGER:
v, err := strconv.ParseInt(lit, 10, 64)
if err != nil {
return nil, err
}
value = v
case FLOAT:
v, err := strconv.ParseFloat(lit, 64)
if err != nil {
return nil, err
}
value = v
case LBRACK:
v, err := p.parseList()
if err != nil {
return nil, err
}
value = v
default:
return nil, parseErrorf(pos, "invalid argument value: %q", lit)
}
// Ensure key doesn't already exist.
if _, ok := args[key]; ok {
return nil, parseErrorf(pos, "argument key already used: %s", key)
}
// If op is specified then create a condition.
if op != 0 {
value = &Condition{Op: op, Value: value}
}
// Add key/value pair to arguments.
args[key] = value
// Exit if closing paren.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
p.unscan(1)
return args, nil
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
}
}
}
// parseList parses a list of primitives. This is used by the TopN() filters.
func (p *Parser) parseList() ([]interface{}, error) {
var values []interface{}
for {
// Read next value.
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
values = append(values, true)
} else if lit == "false" {
values = append(values, false)
} else {
values = append(values, lit)
}
case STRING:
values = append(values, lit)
case INTEGER:
v, err := strconv.ParseInt(lit, 10, 64)
if err != nil {
return nil, err
}
values = append(values, v)
default:
return nil, parseErrorf(pos, "invalid list value: %q", lit)
}
// Expect a comma or closing bracket next.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma, found %q", lit)
}
}
return values, nil
}
// scan returns the next token from the scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
// scanIgnoreWhitespace returns the next non-whitespace token from the scanner.
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
tok, pos, lit = p.scan()
if tok == WS {
tok, pos, lit = p.scan()
}
return
}
// unscan returns the last n tokens back to the scanner.
func (p *Parser) unscan(n int) {
for i := 0; i < n; i++ {
p.scanner.unscan()
}
}
// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner.
func (p *Parser) unscanIgnoreWhitespace(n int) {
for i := 0; i < n; {
p.scanner.unscan()
if tok, _, _ := p.scanner.curr(); tok != WS {
i++
}
}
}
// expect returns an error if the next token is not exp.
func (p *Parser) expect(exp Token) error {
if tok, pos, lit := p.scan(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// pos returns the current position.
func (p *Parser) pos() Pos { return p.scanner.pos() }
// ParseError represents an error that occurred while parsing a PQL query.
type ParseError struct {
Message string
Pos Pos
}
// Error returns a string representation of e.
func (e *ParseError) Error() string {
return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1)
}
// parseErrorf returns a formatted parse error.
func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
return &ParseError{
Message: fmt.Sprintf(format, args...),
Pos: pos,
return nil, errors.Wrap(err, "parsing")
}
p.Execute()
return &p.Query, nil
}

10
pql/parser_test.go
View file

@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) {
// Parse with both child calls and arguments.
t.Run("ChildrenAndArguments", func(t *testing.T) {
q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`)
q, err := pql.ParseString(`TopN(f, Bitmap(id=100, field=other), n=3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
@ -145,7 +145,7 @@ func TestParser_Parse(t *testing.T) {
Name: "Bitmap",
Args: map[string]interface{}{"id": int64(100), "field": "other"},
}},
Args: map[string]interface{}{"n": int64(3), "field": "f"},
Args: map[string]interface{}{"n": int64(3), "_field": "f"},
},
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
@ -154,15 +154,15 @@ func TestParser_Parse(t *testing.T) {
// Parse a list argument.
t.Run("ListArgument", func(t *testing.T) {
q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`)
q, err := pql.ParseString(`TopN(f, ids=[0,10,30])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "TopN",
Args: map[string]interface{}{
"field": "f",
"ids": []interface{}{int64(0), int64(10), int64(30)},
"_field": "f",
"ids": []interface{}{int64(0), int64(10), int64(30)},
},
},
) {

75
pql/pql.peg Normal file
View file

@ -0,0 +1,75 @@
package pql
type PQL Peg {
Query
}
Calls <- whitesp (Call whitesp)* !.
Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()}
/ 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()}
/ 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
/ 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
/ 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()}
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
allargs <- Call (comma Call)* (comma args)? / args / sp
args <- arg (comma args)? sp
arg <- ( field sp '=' sp value
/ field sp COND sp value
)
COND <- ( '><' { p.addBTWN() }
/ '<=' { p.addLTE() }
/ '>=' { p.addGTE() }
/ '==' { p.addEQ() }
/ '!=' { p.addNEQ() }
/ '<' { p.addLT() }
/ '>' { p.addGT() }
)
conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])}
condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])}
condfield <- <fieldExpr> sp {p.condAdd(buffer[begin:end])}
timerange <- field sp '=' sp value comma <timestampfmt> {p.addPosStr("_start", buffer[begin:end])} comma <timestampfmt> {p.addPosStr("_end", buffer[begin:end])}
value <- ( item
/ lbrack { p.startList() } list rbrack { p.endList() }
)
list <- item (comma list)?
item <- ( 'null' &(comma / sp close) { p.addVal(nil) }
/ 'true' &(comma / sp close) { p.addVal(true) }
/ 'false' &(comma / sp close) { p.addVal(false) }
/ < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) }
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) }
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
/ '"' < doublequotedstring > '"' { p.addVal(buffer[begin:end]) }
/ '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) }
)
doublequotedstring <- ( [^"\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )*
singlequotedstring <- ( [^'\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )*
fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' )*
field <- <fieldExpr / reserved> { p.addField(buffer[begin:end]) }
reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field')
posfield <- <fieldExpr> { p.addPosStr("_field", buffer[begin:end]) }
uint <- [1-9] [0-9]* / '0'
uintrow <- <uint>{p.addPosNum("_row", buffer[begin:end])}
col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
/ '"' <doublequotedstring> '"' {p.addPosStr("_col", buffer[begin:end])}
)
open <- '(' sp
close <- ')' sp
sp <- ( ' ' / '\t' )*
comma <- sp ',' whitesp
lbrack <- '[' sp
rbrack <- sp ']' sp
whitesp <- ( ' ' / '\t' / '\n' )*
IDENT <- [[A-Z]] ([[A-Z]] / [0-9])*
timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9]
timestampfmt <- '"' timestampbasicfmt '"' / '\'' timestampbasicfmt '\'' / timestampbasicfmt
timestamp <- <timestampfmt> {p.addPosStr("_timestamp", buffer[begin:end])}

2843
pql/pql.peg.go Normal file
View file

File diff suppressed because it is too large Load diff

526
pql/pqlpeg_test.go Normal file
View file

@ -0,0 +1,526 @@
package pql
import (
"reflect"
"strconv"
"testing"
)
func TestPEG(t *testing.T) {
p := PQL{Buffer: `
SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]}
p.Init()
err := p.Parse()
if err != nil {
t.Fatalf("parse error: %v", err)
}
p.Execute()
p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`}
p.Init()
err = p.Parse()
if err == nil {
t.Fatalf("should have been an error because of the interior unescaped double quote")
}
q, err := ParseString("TopN(blah, Bitmap(id==other), field=f, n=0)")
if err != nil {
t.Fatalf("should have parsed: %v", err)
}
if q.String() != `TopN(Bitmap(id == "other"), _field="blah", field="f", n=0)` {
t.Fatalf("Failed, got: %s", q)
}
q, err = ParseString("C(a=falsen0)")
if err != nil {
t.Fatalf("falsen0 should have been parsed as a string")
}
q, err = ParseString("Bitmap(row=4, did==other)")
if err != nil {
t.Fatalf("should have parsed: %v", err)
}
if q.String() != `Bitmap(did == "other", row=4)` {
t.Fatalf("got %s", q)
}
}
func TestOldPQL(t *testing.T) {
_, err := ParseString(`SetBit(f=11, col=1)`)
if err != nil {
t.Fatalf("should have parsed: %v", err)
}
}
func TestPEGWorking(t *testing.T) {
tests := []struct {
name string
input string
ncalls int
}{
{
name: "Empty",
input: "",
ncalls: 0},
{
name: "Set",
input: "Set(2, f=10)",
ncalls: 1},
{
name: "SetTime",
input: "Set(2, f=1, 1999-12-31T00:00)",
ncalls: 1},
{
name: "DoubleSet",
input: "Set(1, a=4)Set(2, a=4)",
ncalls: 2},
{
name: "DoubleSetSpc",
input: "Set(1, a=4) Set(2, a=4)",
ncalls: 2},
{
name: "DoubleSetNewline",
input: "Set(1, a=4) \n Set(2, a=4)",
ncalls: 2},
{
name: "SetWithArbCall",
input: "Set(1, a=4)Blerg(z=ha)",
ncalls: 2},
{
name: "SetArbSet",
input: "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
ncalls: 3},
{
name: "ArbSetArb",
input: "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
ncalls: 3},
{
name: "SetStringArg",
input: "Set(1, a=zoom)",
ncalls: 1},
{
name: "SetManyArgs",
input: "Set(1, a=4, b=5)",
ncalls: 1},
{
name: "SetManyMixedArgs",
input: "Set(1, a=4, bsd=haha)",
ncalls: 1},
{
name: "SetTimestamp",
input: "Set(1, a=4, 2017-04-03T19:34)",
ncalls: 1},
{
name: "Union()",
input: "Union()",
ncalls: 1},
{
name: "UnionOneRow",
input: "Union(Row(a=1))",
ncalls: 1},
{
name: "UnionTwoRows",
input: "Union(Row(a=1), Row(z=44))",
ncalls: 1},
{
name: "UnionNested",
input: "Union(Intersect(Row(), Union(Row(), Row())), Row())",
ncalls: 1},
{
name: "TopN no args",
input: "TopN(boondoggle)",
ncalls: 1},
{
name: "TopN with args",
input: "TopN(boon, doggle=9)",
ncalls: 1},
{
name: "double quoted args",
input: `B(a="zm''e")`,
ncalls: 1},
{
name: "single quoted args",
input: `B(a='zm""e')`,
ncalls: 1},
{
name: "SetRowAttrs",
input: "SetRowAttrs(blah, 9, a=47)",
ncalls: 1},
{
name: "SetRowAttrs2args",
input: "SetRowAttrs(blah, 9, a=47, b=bval)",
ncalls: 1},
{
name: "SetColumnAttrs",
input: "SetColumnAttrs(9, a=47)",
ncalls: 1},
{
name: "SetColumnAttrs2args",
input: "SetColumnAttrs(9, a=47, b=bval)",
ncalls: 1},
{
name: "Clear",
input: "Clear(1, a=53)",
ncalls: 1},
{
name: "Clear2args",
input: "Clear(1, a=53, b=33)",
ncalls: 1},
{
name: "TopN",
input: "TopN(myfield, n=44)",
ncalls: 1},
{
name: "TopNBitmap",
input: "TopN(myfield, Row(a=47), n=10)",
ncalls: 1},
{
name: "RangeLT",
input: "Range(a < 4)",
ncalls: 1},
{
name: "RangeGT",
input: "Range(a > 4)",
ncalls: 1},
{
name: "RangeLTE",
input: "Range(a <= 4)",
ncalls: 1},
{
name: "RangeGTE",
input: "Range(a >= 4)",
ncalls: 1},
{
name: "RangeEQ",
input: "Range(a == 4)",
ncalls: 1},
{
name: "RangeNEQ",
input: "Range(a != null)",
ncalls: 1},
{
name: "RangeLTLT",
input: "Range(4 < a < 9)",
ncalls: 1},
{
name: "RangeLTLTE",
input: "Range(4 < a <= 9)",
ncalls: 1},
{
name: "RangeLTELT",
input: "Range(4 <= a < 9)",
ncalls: 1},
{
name: "RangeLTELTE",
input: "Range(4 <= a <= 9)",
ncalls: 1},
{
name: "RangeTime",
input: "Range(a=4, 2010-07-04T00:00, 2010-08-04T00:00)",
ncalls: 1},
{
name: "RangeTimeQuotes",
input: `Range(a=4, '2010-07-04T00:00', "2010-08-04T00:00")`,
ncalls: 1},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
q, err := ParseString(test.input)
if err != nil {
t.Fatalf("parsing query '%s': %v", test.input, err)
}
if len(q.Calls) != test.ncalls {
t.Fatalf("wrong number of calls for '%s': %#v", test.input, q.Calls)
}
})
}
}
func TestPEGErrors(t *testing.T) {
tests := []struct {
name string
input string
}{
{
name: "SetNoParens",
input: "Set"},
{
name: "SetBadTimestamp",
input: "Set(1, a=4, 2017-94-03T19:34)"},
{
name: "SetTimestampNoArg",
input: "Set(1, 2017-04-03T19:34)"},
{
name: "SetStartingComma",
input: "Set(, 1, a=4)"},
{
name: "StartinCommaArb",
input: "Zeeb(, a=4)"},
{
name: "SetRowAttrs0args",
input: "SetRowAttrs(blah, 9)"},
{
name: "Clear0args",
input: "Clear(9)"},
{
name: "RangeTimeGT",
input: "Range(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"},
{
name: "RangeTimeOneStamp",
input: "Range(a=4, 2010-07-04T00:00)"},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
q, err := ParseString(test.input)
if err == nil {
t.Fatalf("parsing query '%s' - expected error, got: %s", test.input, q)
}
})
}
}
func TestPQLDeepEquality(t *testing.T) {
tests := []struct {
name string
call string
exp *Call
}{
{
name: "Set",
call: "Set(1, a=7, 2010-07-08T14:44)",
exp: &Call{
Name: "Set",
Args: map[string]interface{}{
"a": int64(7),
"_col": int64(1),
"_timestamp": "2010-07-08T14:44",
},
}},
{
name: "SetRowAttrs",
call: "SetRowAttrs(myfield, 9, z=4)",
exp: &Call{
Name: "SetRowAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_field": "myfield",
"_row": int64(9),
},
}},
{
name: "SetColumnAttrs",
call: "SetColumnAttrs(9, z=4)",
exp: &Call{
Name: "SetColumnAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_col": int64(9),
},
}},
{
name: "Clear",
call: "Clear(1, a=7)",
exp: &Call{
Name: "Clear",
Args: map[string]interface{}{
"a": int64(7),
"_col": int64(1),
},
}},
{
name: "TopN",
call: "TopN(myfield, Row(), a=7)",
exp: &Call{
Name: "TopN",
Args: map[string]interface{}{
"a": int64(7),
"_field": "myfield",
},
Children: []*Call{
{Name: "Row"},
},
}},
{
name: "RangeEQ",
call: "Range(a==7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: EQ,
Value: int64(7),
},
},
}},
{
name: "RangeLT",
call: "Range(a<7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: LT,
Value: int64(7),
},
},
}},
{
name: "RangeLTE",
call: "Range(a<=7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: LTE,
Value: int64(7),
},
},
}},
{
name: "RangeGTE",
call: "Range(a>=7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: GTE,
Value: int64(7),
},
},
}},
{
name: "RangeGT",
call: "Range(a>7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: GT,
Value: int64(7),
},
},
}},
{
name: "RangeNEQ",
call: "Range(a!=null)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: NEQ,
Value: nil,
},
},
}},
{
name: "RangeLTELT",
call: "Range(4 <= a < 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(4), int64(9)},
},
},
}},
{
name: "RangeLTLT",
call: "Range(4 < a < 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(5), int64(9)},
},
},
}},
{
name: "RangeLTELTE",
call: "Range(4 <= a <= 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(4), int64(10)},
},
},
}},
{
name: "RangeLTLTE",
call: "Range(4 < a <= 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(5), int64(10)},
},
},
}},
{
name: "Sum",
call: "Sum(field=f)",
exp: &Call{
Name: "Sum",
Args: map[string]interface{}{
"field": "f",
},
}},
{
name: "SumChild",
call: "Sum(Row(), field=f)",
exp: &Call{
Name: "Sum",
Args: map[string]interface{}{
"field": "f",
},
Children: []*Call{
{Name: "Row"},
},
}},
{
name: "MinChild",
call: "Min(Row(), field=f)",
exp: &Call{
Name: "Min",
Args: map[string]interface{}{
"field": "f",
},
Children: []*Call{
{Name: "Row"},
},
}},
{
name: "MaxChild",
call: "Max(Row(), field=f)",
exp: &Call{
Name: "Max",
Args: map[string]interface{}{
"field": "f",
},
Children: []*Call{
{Name: "Row"},
},
}},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
q, err := ParseString(test.call)
if err != nil {
t.Fatalf("parsing query '%s': %v", test.call, err)
}
if !reflect.DeepEqual(test.exp, q.Calls[0]) {
t.Fatalf("unexpected call:\n%s\ninstead of:\n%s\n'%#v'\ninstead of:\n'%#v'", q.Calls[0], test.exp, q.Calls[0], test.exp)
}
})
}
}

303
pql/scanner.go
View file

@ -1,303 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pql
import (
"bufio"
"bytes"
"io"
"unicode"
)
// Scanner represents a PQL lexical scanner.
type Scanner struct {
r io.RuneScanner
pos Pos
}
// NewScanner returns a new instance of Scanner.
func NewScanner(r io.Reader) *Scanner {
return &Scanner{r: bufio.NewReader(r)}
}
// Scan returns the next token and position from the underlying reader.
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
pos = s.pos
// Read next code point.
ch := s.read()
// If we see whitespace then consume all contiguous whitespace.
// If we see a letter, or certain acceptable special characters, then consume
// as an ident or reserved word. If we see quotes, then scan as string.
if isWhitespace(ch) {
s.unread()
return s.scanWhitespace()
} else if isIdentFirstChar(ch) {
s.unread()
return s.scanIdent()
} else if isDigit(ch) || ch == '-' {
s.unread()
return s.scanNumber()
} else if ch == '"' || ch == '\'' {
s.unread()
return s.scanString()
}
// Otherwise parse individual characters.
switch ch {
case eof:
return EOF, pos, ""
case '=':
if next := s.read(); next == '=' {
return EQ, pos, "=="
}
s.unread()
return ASSIGN, pos, string(ch)
case '!':
if next := s.read(); next == '=' {
return NEQ, pos, "!="
}
s.unread()
return ASSIGN, pos, string(ch)
case '<':
if next := s.read(); next == '=' {
return LTE, pos, "<="
}
s.unread()
return LT, pos, string(ch)
case '>':
next := s.read()
if next == '=' {
return GTE, pos, ">="
} else if next == '<' {
return BETWEEN, pos, "><"
}
s.unread()
return GT, pos, string(ch)
case ',':
return COMMA, pos, string(ch)
case '(':
return LPAREN, pos, string(ch)
case ')':
return RPAREN, pos, string(ch)
case '[':
return LBRACK, pos, string(ch)
case ']':
return RBRACK, pos, string(ch)
default:
return ILLEGAL, pos, string(ch)
}
}
// read returns the next code point from the underlying reader and updates the pos.
func (s *Scanner) read() rune {
// Read next rune from underlying reader.
ch, _, err := s.r.ReadRune()
if err != nil {
return eof
}
// Update position information.
if ch == '\n' {
s.pos.Line++
s.pos.Char = 0
} else {
s.pos.Char++
}
return ch
}
// unread pushes the previously read rune back onto the reader.
func (s *Scanner) unread() {
if s.pos.Char == 0 {
s.pos.Line--
} else {
s.pos.Char--
}
s.r.UnreadRune()
}
// scanWhitespace consumes the current rune and all contiguous whitespace.
func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isWhitespace(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
return WS, pos, buf.String()
}
func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isIdentChar(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
lit = buf.String()
// If the literal matches a keyword then return that keyword.
if tok = Lookup(lit); tok != IDENT {
return tok, pos, lit
}
return IDENT, pos, lit
}
// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character.
func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) {
pos = s.pos
tok = INTEGER
var buf bytes.Buffer
var seenDot bool
first := true
for {
ch := s.read()
if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') {
s.unread()
break
}
if ch == '.' {
seenDot = true
tok = FLOAT
}
buf.WriteRune(ch)
first = false
}
return tok, pos, buf.String()
}
// scanString consumes a single-quoted or double-quoted string.
func (s *Scanner) scanString() (tok Token, pos Pos, lit string) {
pos = s.pos
// This must be either a single- or double-quote.
ending := s.read()
var buf bytes.Buffer
for {
ch := s.read()
if ch == ending {
break
} else if ch == '\n' || ch == eof {
return BADSTRING, pos, buf.String()
} else if ch == '\\' {
next := s.read()
if next == 'n' {
buf.WriteRune('\n')
} else if next == '\\' {
buf.WriteRune('\\')
} else if next == '"' {
buf.WriteRune('"')
} else if next == '\'' {
buf.WriteRune('\'')
} else {
return BADSTRING, pos, buf.String()
}
} else {
buf.WriteRune(ch)
}
}
return STRING, pos, buf.String()
}
// bufScanner represents a wrapper for scanner to add a buffer.
// It provides a fixed-length circular buffer that can be unread.
type bufScanner struct {
s *Scanner
i int // buffer index
n int // buffer size
buf [8]struct {
tok Token
pos Pos
lit string
}
}
// newBufScanner returns a new buffered scanner for a reader.
func newBufScanner(r io.Reader) *bufScanner {
return &bufScanner{s: NewScanner(r)}
}
// Scan reads the next token from the scanner.
func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) {
// If we have unread tokens then read them off the buffer first.
if s.n > 0 {
s.n--
return s.curr()
}
// Move buffer position forward and save the token.
s.i = (s.i + 1) % len(s.buf)
buf := &s.buf[s.i]
buf.tok, buf.pos, buf.lit = s.s.Scan()
return s.curr()
}
// unscan pushes the previously token back onto the buffer.
func (s *bufScanner) unscan() { s.n++ }
// curr returns the last read token.
func (s *bufScanner) curr() (tok Token, pos Pos, lit string) {
buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)]
return buf.tok, buf.pos, buf.lit
}
// pos returns the current position.
func (s *bufScanner) pos() Pos {
_, pos, _ := s.curr()
return pos
}
// isWhitespace returns true if the rune a Unicode space character.
func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) }
// isLetter returns true if the rune is a letter.
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
// isDigit returns true if the rune is a digit.
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
// isIdentChar returns true if the rune can be used in an unquoted identifier.
func isIdentChar(ch rune) bool {
return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.'
}
// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }
const eof = rune(0)

74
pql/scanner_test.go
View file

@ -1,74 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pql_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/pql"
)
func TestScanner_Scan(t *testing.T) {
var tests = []struct {
name string
s string
tok pql.Token
lit string
pos pql.Pos
}{
// Special tokens (EOF, ILLEGAL, WS)
{name: "EOF", s: ``, tok: pql.EOF},
{name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`},
{name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "},
{name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"},
{name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"},
{name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`},
{name: "EQ", s: `==`, tok: pql.EQ, lit: `==`},
{name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`},
{name: "LT", s: `<`, tok: pql.LT, lit: `<`},
{name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`},
{name: "GT", s: `>`, tok: pql.GT, lit: `>`},
{name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`},
{name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`},
{name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`},
{name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`},
{name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`},
{name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`},
{name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`},
{name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`},
{name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`},
{name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`},
{name: "ALL", s: `all`, tok: pql.ALL, lit: `all`},
{name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive
}
for i, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := pql.NewScanner(strings.NewReader(tt.s))
tok, pos, lit := s.Scan()
if tt.tok != tok {
t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit)
} else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char {
t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos)
} else if tt.lit != lit {
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit)
}
})
}
}

58
pql/token.go
View file

@ -14,28 +14,12 @@
package pql
import "strings"
// Token is a lexical token of the PQL language.
type Token int
const (
// Special tokens
ILLEGAL Token = iota
EOF
WS
literal_beg
IDENT // main
STRING // "foo"
BADSTRING // bad escape or unclosed string
INTEGER // 12345
FLOAT // 100.2
literal_end
keyword_beg
ALL
keyword_end
ASSIGN // =
EQ // ==
@ -45,23 +29,10 @@ const (
GT // >
GTE // >=
BETWEEN // ><
COMMA // ,
LPAREN // (
RPAREN // )
LBRACK // (
RBRACK // )
)
var tokens = [...]string{
ILLEGAL: "ILLEGAL",
EOF: "EOF",
WS: "WS",
IDENT: "IDENT",
INTEGER: "INTEGER",
FLOAT: "FLOAT",
ALL: "ALL",
ASSIGN: "=",
EQ: "==",
@ -71,20 +42,6 @@ var tokens = [...]string{
GT: ">",
GTE: ">=",
BETWEEN: "><",
COMMA: ",",
LPAREN: "(",
RPAREN: ")",
LBRACK: "(",
RBRACK: ")",
}
var keywords map[string]Token
func init() {
keywords = make(map[string]Token)
for tok := keyword_beg + 1; tok < keyword_end; tok++ {
keywords[strings.ToLower(tokens[tok])] = tok
}
}
// String returns the string representation of the token.
@ -94,18 +51,3 @@ func (tok Token) String() string {
}
return ""
}
// Lookup returns the token associated with a given string.
func Lookup(ident string) Token {
if tok, ok := keywords[strings.ToLower(ident)]; ok {
return tok
}
return IDENT
}
// Pos specifies the line and character position of a token.
// The Char and Line are both zero-based indexes.
type Pos struct {
Line int
Char int
}

7
roaring/containers.go
View file

@ -132,6 +132,13 @@ func (sc *SliceContainers) Count() uint64 {
return n
}
func (sc *SliceContainers) Reset() {
sc.keys = sc.keys[:0]
sc.containers = sc.containers[:0]
sc.lastContainer = nil
sc.lastKey = 0
}
func (sc *SliceContainers) seek(key uint64) (int, bool) {
i := search64(sc.keys, key)
found := true

5
roaring/roaring.go
View file

@ -94,6 +94,8 @@ type Containers interface {
// container is found at key.
Iterator(key uint64) (citer ContainerIterator, found bool)
Count() uint64
//Reset will clear the containers collection to allow for recycling during snapshot
Reset()
}
type ContainerIterator interface {
@ -631,7 +633,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
keyN := binary.LittleEndian.Uint32(data[4:8])
headerSize := headerBaseSize
b.Containers.Reset()
// Descriptive header section: Read container keys and cardinalities.
for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
b.Containers.PutContainerValues(
@ -688,6 +690,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
// FIXME(benbjohnson): return error with position so file can be trimmed.
return err
}
opr.apply(b)
// Increase the op count.

14
server/cluster_test.go
View file

@ -92,8 +92,8 @@ func TestMain_SendReceiveMessage(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
Set(1, f=1)
Set(2400000, f=1)
`); err != nil {
t.Fatal(err)
}
@ -259,8 +259,8 @@ func TestClusterResize_AddNode(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=1300000)
Set(1, f=1)
Set(1300000, f=1)
`); err != nil {
t.Fatal(err)
}
@ -311,8 +311,8 @@ func TestClusterResize_AddNode(t *testing.T) {
// Write data on first node. Note that no data is placed on slice 1.
if _, err := m0.Query("i", "", `
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
Set(1, f=1)
Set(2400000, f=1)
`); err != nil {
t.Fatal(err)
}
@ -466,7 +466,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// TODO: Deterministic node IDs would ensure consistent results
setColumns := ""
for i := 0; i < 20; i++ {
setColumns += fmt.Sprintf("SetBit(row=1, field=\"f\", col=%d) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setColumns); err != nil {

2
server/config.go
View file

@ -47,7 +47,7 @@ type Config struct {
Bind string `toml:"bind"`
// MaxWritesPerRequest limits the number of mutating commands that can be in
// a single request to the server. This includes SetBit, ClearBit,
// a single request to the server. This includes Set, Clear,
// SetRowAttrs & SetColumnAttrs.
MaxWritesPerRequest int `toml:"max-writes-per-request"`

48
server/server_test.go
View file

@ -49,7 +49,7 @@ func TestMain_Set_Quick(t *testing.T) {
t.Fatal(err)
}
// Execute SetBit() commands.
// Execute Set() commands.
for _, cmd := range cmds {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
@ -57,7 +57,7 @@ func TestMain_Set_Quick(t *testing.T) {
if err := client.CreateField(context.Background(), "i", cmd.Field, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists {
t.Fatal(err)
}
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, field=%q, col=%d)`, cmd.ID, cmd.Field, cmd.ColumnID)); err != nil {
if _, err := m.Query("i", "", fmt.Sprintf(`Set(%d, %s=%d)`, cmd.ColumnID, cmd.Field, cmd.ID)); err != nil {
t.Fatal(err)
}
}
@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -132,36 +132,36 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Set columns on different rows in different fields.
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, field="x", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, field="z", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=3, field="neg", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `Set(100, neg=3)`); err != nil {
t.Fatal(err)
}
// Set row attributes.
if _, err := m.Query("i", "", `SetRowAttrs(row=1, field="x", x=100)`); err != nil {
if _, err := m.Query("i", "", `SetRowAttrs(x, 1, x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="x", x=-200)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(x, 2, x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="z", x=300)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(z, 2, x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=3, field="neg", x=-0.44)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(neg, 3, x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
if res, err := m.Query("i", "", `Row(x=2)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, field="neg")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(neg=3)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
if res, err := m.Query("i", "", `Row(x=2)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -205,19 +205,19 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Set columns on row.
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=101)`); err != nil {
} else if _, err := m.Query("i", "", `Set(101, x=1)`); err != nil {
t.Fatal(err)
}
// Set column attributes.
if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
if _, err := m.Query("i", "", `SetColumnAttrs(100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
@ -279,7 +279,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
data = append(data, fmt.Sprintf(`SetBit(row=%d, field="f", col=%d)`, rowID, columnID))
data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID))
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
@ -296,7 +296,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster {
res, err := m.Query("i", "", `TopN(field="f")`)
res, err := m.Query("i", "", `TopN(f)`)
if err != nil {
t.Fatal(err)
}

10
stats_test.go
View file

@ -128,8 +128,8 @@ func TestStatsCount_Bitmap(t *testing.T) {
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "Bitmap" {
t.Errorf("Expected Bitmap, Results %s", name)
if name != "Row" {
t.Errorf("Expected Row, Results %s", name)
}
if tags[0] != "index:d" {
@ -139,7 +139,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(field=f, row=0)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Row(f=0)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -169,7 +169,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -200,7 +200,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, field=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(10, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {