scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into fix-translate

Browse source
This commit is contained in:
Matt Jaffee 2018-06-25 12:33:52 -05:00
commit b2dd9c50ca
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
37 changed files with 4978 additions and 2240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-pql install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-peg test
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@ -92,8 +92,11 @@ generate-protoc: require-protoc require-protoc-gen-gofast
generate-stringer:
go generate github.com/pilosa/pilosa
generate-pql: require-peg
cd pql && peg -inline pql.peg && cd ..
# `go generate` all needed packages
generate: generate-protoc generate-stringer
generate: generate-protoc generate-stringer generate-pql
# Create Docker image from Dockerfile
docker:
@ -128,7 +131,10 @@ require-protoc-gen-gofast:
require-protoc:
$(call require,protoc)
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer
require-peg:
$(call require,peg)
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-peg
install-dep:
go get -u github.com/golang/dep/cmd/dep
@ -141,3 +147,6 @@ install-protoc-gen-gofast:
install-protoc:
@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
install-peg:
go get github.com/pointlander/peg

32
api.go
View file

@ -46,16 +46,43 @@ type API struct {
Cluster *Cluster
TranslateStore TranslateStore
Logger Logger
server *Server
}
// APIOption is a functional option type for pilosa.API
type APIOption func(*API) error
func OptAPIServer(s *Server) APIOption {
return func(a *API) error {
a.server = s
a.Executor = s.executor
a.TranslateStore = s.translateFile
a.Holder = s.holder
a.Broadcaster = s
a.BroadcastHandler = s
a.StatusHandler = s
a.Cluster = s.Cluster
a.Logger = s.logger
return nil
}
}
// NewAPI returns a new API instance.
func NewAPI() *API {
return &API{
func NewAPI(opts ...APIOption) (*API, error) {
api := &API{
Broadcaster: NopBroadcaster,
//BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop
//StatusHandler: NopStatusHandler, // TODO: implement the nop
Logger: NopLogger,
}
for _, opt := range opts {
err := opt(api)
if err != nil {
return nil, errors.Wrap(err, "applying option")
}
}
return api, nil
}
// validAPIMethods specifies the api methods that are valid for each
@ -658,7 +685,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
if err != nil {
return errors.Wrap(err, "getting field")
}
// Import into fragment.
err = field.ImportValue(req.ColumnIDs, req.Values)
if err != nil {

4
cluster.go
View file

@ -229,7 +229,7 @@ type Cluster struct {
// Threshold for logging long-running queries
LongQueryTime time.Duration
// Maximum number of SetBit() or ClearBit() commands per request.
// Maximum number of Set() or Clear() commands per request.
MaxWritesPerRequest int
// EventReceiver receives NodeEvents pertaining to node membership.
@ -914,7 +914,7 @@ func (c *Cluster) open() error {
return fmt.Errorf("sending restart NodeJoin: %v", err)
}
c.Logger.Printf("wait for joining to complete")
c.Logger.Printf("%v wait for joining to complete", c.Node.ID)
<-c.joining
c.Logger.Printf("joining has completed")
}

2
cmd/server_test.go
View file

@ -15,7 +15,6 @@
package cmd_test
import (
"errors"
"io/ioutil"
"strings"
"testing"
@ -24,6 +23,7 @@ import (
"github.com/pilosa/pilosa/cmd"
_ "github.com/pilosa/pilosa/test"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
)
func TestServerHelp(t *testing.T) {

18
ctl/export_test.go
View file

@ -44,22 +44,16 @@ func TestExportCommand_Validation(t *testing.T) {
}
func TestExportCommand_Run(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewExportCommand(stdin, stdout, stderr)
hostport := cmd.Server.URI.HostPort()
cm.Host = hostport
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader("")))
cm.Index = "i"
cm.Field = "f"

85
ctl/import_test.go
View file

@ -51,7 +51,6 @@ func TestImportCommand_Validation(t *testing.T) {
}
func TestImportCommand_Run(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
@ -62,15 +61,8 @@ func TestImportCommand_Run(t *testing.T) {
t.Fatal(err)
}
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
cmd := test.MustRunMainWithCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
cm.Index = "i"
cm.Field = "f"
@ -84,7 +76,6 @@ func TestImportCommand_Run(t *testing.T) {
// Ensure that the ImportValue path runs.
func TestImportCommand_RunValue(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
@ -95,18 +86,11 @@ func TestImportCommand_RunValue(t *testing.T) {
t.Fatal(err)
}
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
cm.Index = "i"
cm.Field = "f"
@ -118,20 +102,12 @@ func TestImportCommand_RunValue(t *testing.T) {
}
func TestImportCommand_InvalidFile(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cmd := test.MustRunMainWithCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = s.Host()
cm.Host = cmd.Server.URI.HostPort()
cm.Index = "i"
cm.Field = "f"
file, err := ioutil.TempFile("", "import.csv")
@ -198,3 +174,48 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}
func TestImportCommand_BugOverwriteValue(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import-value.csv")
file.Write([]byte("0,17\n"))
ctx := context.Background()
if err != nil {
t.Fatal(err)
}
cm.Host = cmd.Server.Addr().String()
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
cm.Index = "i"
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
file.Close()
file, err = ioutil.TempFile("", "import-value2.csv")
file.Write([]byte("0,16\n"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
file.Close()
file, err = ioutil.TempFile("", "import-value3.csv")
file.Write([]byte("0,19\n"))
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
t.Fatalf("Import Run with values doesn't work: %s", err)
}
}

6
enterprise/b/containers_btree.go
View file

@ -160,6 +160,12 @@ func (btc *BTreeContainers) Size() int {
return btc.tree.Len()
}
func (btc *BTreeContainers) Reset() {
btc.tree = TreeNew(cmp)
btc.lastKey = 0
btc.lastContainer = nil
}
func (btc *BTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterator, found bool) {
e, ok := btc.tree.Seek(key)
if ok {

176
executor.go
View file

@ -48,7 +48,7 @@ type Executor struct {
// Client used for remote requests.
client InternalQueryClient
// Maximum number of SetBit() or ClearBit() commands per request.
// Maximum number of Set() or Clear() commands per request.
MaxWritesPerRequest int
// Stores key/id translation data.
@ -178,12 +178,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
case "Max":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeMax(ctx, index, c, slices, opt)
case "ClearBit":
case "Clear":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeCount(ctx, index, c, slices, opt)
case "SetBit":
case "Set":
return e.executeSetBit(ctx, index, c, opt)
case "SetValue":
return nil, e.executeSetValue(ctx, index, c, opt)
@ -340,17 +340,17 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
return nil, err
}
// Attach attributes for Bitmap() calls.
// Attach attributes for Row() calls.
// If the column label is used then return column attributes.
// If the row label is used then return bitmap attributes.
row, _ := other.(*Row)
if c.Name == "Bitmap" {
if c.Name == "Row" {
if opt.ExcludeRowAttrs {
row.Attrs = map[string]interface{}{}
} else {
idx := e.Holder.Index(index)
if idx != nil {
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
if columnID, ok, err := c.UintArg("_" + columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
return nil, errors.Wrap(err, "getting column attrs")
@ -359,9 +359,10 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
} else if err != nil {
return nil, err
} else {
field, _ := c.Args["field"].(string)
if fr := idx.Field(field); fr != nil {
rowID, _, err := c.UintArg(rowLabel)
// field, _ := c.Args["field"].(string)
fieldName, _ := c.FieldArg()
if fr := idx.Field(fieldName); fr != nil {
rowID, _, err := c.UintArg(fieldName)
if err != nil {
return nil, errors.Wrap(err, "getting row")
}
@ -386,7 +387,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
// executeBitmapCallSlice executes a bitmap call for a single slice.
func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
switch c.Name {
case "Bitmap":
case "Row":
return e.executeBitmapSlice(ctx, index, c, slice)
case "Difference":
return e.executeDifferenceSlice(ctx, index, c, slice)
@ -585,7 +586,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
field, _ := c.Args["field"].(string)
field, _ := c.Args["_field"].(string)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -675,24 +676,24 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
}
// Fetch field & row label based on argument.
field, _ := c.Args["field"].(string)
if field == "" {
field = defaultField
fieldName, err := c.FieldArg()
if err != nil {
return nil, errors.New("Row() argument required: field")
}
f := e.Holder.Field(index, field)
f := e.Holder.Field(index, fieldName)
if f == nil {
return nil, ErrFieldNotFound
}
rowID, rowOK, rowErr := c.UintArg(rowLabel)
rowID, rowOK, rowErr := c.UintArg(fieldName)
if rowErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr)
return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr)
}
if !rowOK {
return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel)
return nil, fmt.Errorf("Row() must specify %v", rowLabel)
}
frag := e.Holder.Fragment(index, field, ViewStandard, slice)
frag := e.Holder.Fragment(index, fieldName, ViewStandard, slice)
if frag == nil {
return NewRow(), nil
}
@ -728,10 +729,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
return e.executeBSIGroupRangeSlice(ctx, index, c, slice)
}
// Parse field, use default if unset.
field, _ := c.Args["field"].(string)
if field == "" {
field = defaultField
// Parse field.
fieldName, err := c.FieldArg()
if err != nil {
return nil, errors.New("Range() argument required: field")
}
// Retrieve column label.
@ -741,13 +742,13 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Retrieve base field.
f := idx.Field(field)
f := idx.Field(fieldName)
if f == nil {
return nil, ErrFieldNotFound
}
// Read row & column id.
rowID, rowOK, err := c.UintArg(rowLabel)
rowID, rowOK, err := c.UintArg(fieldName)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err)
}
@ -756,7 +757,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Parse start time.
startTimeStr, ok := c.Args["start"].(string)
startTimeStr, ok := c.Args["_start"].(string)
if !ok {
return nil, errors.New("Range() start time required")
}
@ -766,7 +767,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Parse end time.
endTimeStr, ok := c.Args["end"].(string)
endTimeStr, ok := c.Args["_end"].(string)
if !ok {
return nil, errors.New("Range() end time required")
}
@ -784,7 +785,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based views.
row := &Row{}
for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, field, view, slice)
f := e.Holder.Fragment(index, fieldName, view, slice)
if f == nil {
continue
}
@ -994,11 +995,11 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
return n, nil
}
// executeClearBit executes a ClearBit() call.
// executeClearBit executes a Clear() call.
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
field, ok := c.Args["field"].(string)
if !ok {
return false, errors.New("ClearBit() field required")
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("Clear() argument required: field")
}
// Retrieve field.
@ -1006,30 +1007,30 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
if idx == nil {
return false, ErrIndexNotFound
}
f := idx.Field(field)
f := idx.Field(fieldName)
if f == nil {
return false, ErrFieldNotFound
}
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading ClearBit() row: %v", err)
return false, fmt.Errorf("reading Clear() row: %v", err)
} else if !ok {
return false, fmt.Errorf("ClearBit() row field '%v' required", rowLabel)
return false, fmt.Errorf("Clear() row argument '%v' required", rowLabel)
}
colID, ok, err := c.UintArg(columnLabel)
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
return false, fmt.Errorf("reading ClearBit() column: %v", err)
return false, fmt.Errorf("reading Clear() column: %v", err)
} else if !ok {
return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel)
return false, fmt.Errorf("Clear() col argument '%v' required", columnLabel)
}
return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt)
}
// executeClearBitField executes a ClearBit() call for a single view.
// executeClearBitField executes a Clear() call for a single view.
func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
@ -1059,11 +1060,11 @@ func (e *Executor) executeClearBitField(ctx context.Context, index string, c *pq
return ret, nil
}
// executeSetBit executes a SetBit() call.
// executeSetBit executes a Set() call.
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
field, ok := c.Args["field"].(string)
if !ok {
return false, errors.New("SetBit() field required: field")
fieldName, err := c.FieldArg()
if err != nil {
return false, errors.New("Set() argument required: field")
}
// Retrieve field.
@ -1071,28 +1072,28 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
if idx == nil {
return false, ErrIndexNotFound
}
f := idx.Field(field)
f := idx.Field(fieldName)
if f == nil {
return false, ErrFieldNotFound
}
// Read fields using labels.
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg(fieldName)
if err != nil {
return false, fmt.Errorf("reading SetBit() row: %v", err)
return false, fmt.Errorf("reading Set() row: %v", err)
} else if !ok {
return false, fmt.Errorf("SetBit() row field '%v' required", rowLabel)
return false, fmt.Errorf("Set() row argument '%v' required", rowLabel)
}
colID, ok, err := c.UintArg(columnLabel)
colID, ok, err := c.UintArg("_" + columnLabel)
if err != nil {
return false, fmt.Errorf("reading SetBit() column: %v", err)
return false, fmt.Errorf("reading Set() column: %v", err)
} else if !ok {
return false, fmt.Errorf("SetBit() column field '%v' required", columnLabel)
return false, fmt.Errorf("Set() column argument '%v' required", columnLabel)
}
var timestamp *time.Time
sTimestamp, ok := c.Args["timestamp"].(string)
sTimestamp, ok := c.Args["_timestamp"].(string)
if ok {
t, err := time.Parse(TimeFormat, sTimestamp)
if err != nil {
@ -1104,7 +1105,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt)
}
// executeSetBitField executes a SetBit() call for a specific view.
// executeSetBitField executes a Set() call for a specific view.
func (e *Executor) executeSetBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
@ -1198,7 +1199,7 @@ func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Cal
// executeSetRowAttrs executes a SetRowAttrs() call.
func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
fieldName, ok := c.Args["field"].(string)
fieldName, ok := c.Args["_field"].(string)
if !ok {
return errors.New("SetRowAttrs() field required")
}
@ -1210,7 +1211,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
}
// Parse labels.
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg("_" + rowLabel)
if err != nil {
return fmt.Errorf("reading SetRowAttrs() row: %v", err)
} else if !ok {
@ -1219,8 +1220,8 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "field")
delete(attrs, rowLabel)
delete(attrs, "_field")
delete(attrs, "_"+rowLabel)
// Set attributes.
if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
@ -1258,7 +1259,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
// Collect attributes by field/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, c := range calls {
field, ok := c.Args["field"].(string)
field, ok := c.Args["_field"].(string)
if !ok {
return nil, errors.New("SetRowAttrs() field required")
}
@ -1269,7 +1270,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
return nil, ErrFieldNotFound
}
rowID, ok, err := c.UintArg(rowLabel)
rowID, ok, err := c.UintArg("_" + rowLabel)
if err != nil {
return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel)
} else if !ok {
@ -1278,8 +1279,8 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "field")
delete(attrs, rowLabel)
delete(attrs, "_field")
delete(attrs, "_"+rowLabel)
// Create field group, if not exists.
fieldMap := m[field]
@ -1348,14 +1349,14 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
return ErrIndexNotFound
}
col, okCol, errCol := c.UintArg(columnLabel)
col, okCol, errCol := c.UintArg("_" + columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetColumnAttrs() col errs: %v found %v", errCol, okCol)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, columnLabel)
delete(attrs, "_"+columnLabel)
delete(attrs, "field")
// Set attributes.
@ -1420,9 +1421,9 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q *
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
v, err = pb.Results[i].N, nil
case "SetBit":
case "Set":
v, err = pb.Results[i].Changed, nil
case "ClearBit":
case "Clear":
v, err = pb.Results[i].Changed, nil
case "SetRowAttrs":
case "SetColumnAttrs":
@ -1493,6 +1494,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64,
case resp := <-ch:
// On error retry against remaining nodes. If an error returns then
// the context will cancel and cause all open goroutines to return.
if resp.err != nil {
// Filter out unavailable nodes.
nodes = Nodes(nodes).Filter(resp.node)
@ -1591,40 +1593,55 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu
}
func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error {
var colKey, rowKey, fieldName string
if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" {
// Positional args in new PQL syntax require special handling here.
colKey = "_" + columnLabel
fieldName, _ = c.FieldArg()
rowKey = fieldName
} else {
colKey = "col"
fieldName = callArgString(c, "field")
rowKey = "row"
}
// Translate column key.
if idx.Keys() {
if c.Args["col"] != nil && !isString(c.Args["col"]) {
return errors.New("'col' value must be a string when index 'keys' option enabled")
if c.Args[colKey] != nil && !isString(c.Args[colKey]) {
return errors.New("column value must be a string when index 'keys' option enabled")
}
if value := callArgString(c, "col"); value != "" {
if value := callArgString(c, colKey); value != "" {
ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value})
if err != nil {
return err
}
c.Args["col"] = ids[0]
fmt.Printf("translated %s to %d in field %s\n", value, ids[0], fieldName)
c.Args[colKey] = ids[0]
}
} else {
if isString(c.Args["col"]) {
if isString(c.Args[colKey]) {
return errors.New("string 'col' value not allowed unless index 'keys' option enabled")
}
}
// Translate row key, if field is specified & key exists.
if fieldName := callArgString(c, "field"); fieldName != "" {
if fieldName != "" {
field := idx.Field(fieldName)
if field == nil {
return ErrFieldNotFound
}
if field.Keys() {
if c.Args["row"] != nil && !isString(c.Args["row"]) {
return errors.New("'row' value must be a string when field 'keys' option enabled")
if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) {
return errors.New("row value must be a string when field 'keys' option enabled")
}
if value := callArgString(c, "row"); value != "" {
if value := callArgString(c, rowKey); value != "" {
ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value})
if err != nil {
return err
}
c.Args["row"] = ids[0]
c.Args[rowKey] = ids[0]
}
} else {
if isString(c.Args["row"]) {
if isString(c.Args[rowKey]) {
return errors.New("string 'row' value not allowed unless field 'keys' option enabled")
}
}
@ -1658,8 +1675,11 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res
}
case []Pair:
if fieldName := callArgString(call, "field"); fieldName != "" {
if fieldName := callArgString(call, "_field"); fieldName != "" {
field := idx.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
}
if field.Keys() {
other := make([]Pair, len(result))
for i := range result {
@ -1727,7 +1747,7 @@ func needsSlices(calls []*pql.Call) bool {
}
for _, call := range calls {
switch call.Name {
case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs":
case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs":
continue
case "Count", "TopN":
return true

401
executor_test.go
View file

@ -27,6 +27,7 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
"github.com/pkg/errors"
)
// Ensure a bitmap query can be executed.
@ -44,9 +45,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -54,7 +55,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
@ -63,7 +64,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit column attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -72,7 +73,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit row attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -93,9 +94,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 10)+
fmt.Sprintf("Set(%d, f=%d)\n", SliceWidth+1, 20),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -116,15 +117,15 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
`SetBit(field=f, row="bar", col="foo")`+"\n"+
`SetBit(field=f, row="baz", col="foo")`+"\n"+
`SetBit(field=f, row="bar", col="bat")`+"\n"+
`SetBit(field=f, row="bbb", col="aaa")`+"\n",
`Set("foo", f="bar")`+"\n"+
`Set("foo", f="baz")`+"\n"+
`Set("bat", f="bar")`+"\n"+
`Set("aaa", f="bbb")`+"\n",
), nil, nil); err != nil {
t.Fatal(err)
}
if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil {
if results, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f="bar")`), nil, nil); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(results, []interface{}{
&pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}},
@ -145,7 +146,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
hldr.SetBit("i", "general", 11, 4)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -177,7 +178,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -207,7 +208,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -240,7 +241,7 @@ func TestExecutor_Execute_Xor(t *testing.T) {
hldr.SetBit("i", "general", 11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -256,7 +257,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr.SetBit("i", "f", 10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
@ -266,22 +267,21 @@ func TestExecutor_Execute_Count(t *testing.T) {
// Ensure a set query can be executed.
func TestExecutor_Execute_SetBit(t *testing.T) {
t.Run("ID", func(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
hldr.SetBit("i", "f", 1, 0)
t.Run("OK", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// set a bit so the view gets created.
hldr.SetBit("i", "f", 1, 0)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
hldr.ClearBit("i", "f", 11, 1)
if n := hldr.Row("i", "f", 11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil {
if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
if !res.Results[0].(bool) {
t.Fatalf("expected column changed")
}
}
@ -289,62 +289,44 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := hldr.Row("i", "f", 11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil {
if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
if res.Results[0].(bool) {
t.Fatalf("expected column unchanged")
}
}
})
t.Run("ErrInvalidColValueType", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row=1, col="foo")`), nil, nil); err == nil || err.Error() != `string 'col' value not allowed unless index 'keys' option enabled` {
t.Fatal(err)
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=1)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` {
t.Fatalf("The error is: '%v'", err)
}
})
t.Run("ErrInvalidRowValueType", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row="bar", col=2)`), nil, nil); err == nil || err.Error() != `string 'row' value not allowed unless field 'keys' option enabled` {
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2, f="bar")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` {
t.Fatal(err)
}
})
})
t.Run("Keys", func(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
t.Run("OK", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
// set a bit so the view gets created.
hldr.SetBit("i", "f", 1, 0)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if n := hldr.Row("i", "f", 11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col="foo")`), nil, nil); err != nil {
if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
if !res.Results[0].(bool) {
t.Fatalf("expected column changed")
}
}
@ -352,47 +334,55 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := hldr.Row("i", "f", 11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col="foo")`), nil, nil); err != nil {
if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
if res.Results[0].(bool) {
t.Fatalf("expected column unchanged")
}
}
})
t.Run("ErrInvalidColValueType", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
if err := index.DeleteField("f"); err != nil {
t.Fatal(err)
}
if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row=1, col=2)`), nil, nil); err == nil || err.Error() != `'col' value must be a string when index 'keys' option enabled` {
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2, f=1)`}); err == nil || errors.Cause(err).Error() != `column value must be a string when index 'keys' option enabled` {
t.Fatal(err)
}
})
t.Run("ErrInvalidRowValueType", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(field=f, row=1, col=2)`), nil, nil); err == nil || err.Error() != `'row' value must be a string when field 'keys' option enabled` {
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1)`}); err == nil || errors.Cause(err).Error() != `row value must be a string when field 'keys' option enabled` {
t.Fatal(err)
}
})
})
}
// Ensure old PQL syntax doesn't break anything too badly.
func TestExecutor_Execute_OldPQL(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// set a bit so the view gets created.
hldr.SetBit("i", "f", 1, 0)
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(frame=f, row=11, col=1)`), nil, nil); err == nil || err.Error() != "unknown call: SetBit" {
t.Fatal("Expected error: 'unknown call: SetBit'")
}
}
// Ensure a SetValue() query can be executed.
func TestExecutor_Execute_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
@ -488,16 +478,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Set two attrs on f/10.
// Also set attrs on other bitmaps and fields to test isolation.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, field=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 200, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(xxx, 10, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -524,15 +514,15 @@ func TestExecutor_Execute_TopN(t *testing.T) {
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=1)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(field=f, row=10, col=0)
SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=other, row=0, col=0)
Set(0, f=0)
Set(1, f=0)
Set(`+strconv.Itoa(SliceWidth)+`, f=0)
Set(`+strconv.Itoa(SliceWidth+2)+`, f=0)
Set(`+strconv.Itoa((5*SliceWidth)+100)+`, f=0)
Set(0, f=10)
Set(`+strconv.Itoa(SliceWidth)+`, f=10)
Set(`+strconv.Itoa(SliceWidth)+`, f=20)
Set(0, other=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -541,7 +531,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: 0, Count: 5},
@ -564,22 +554,22 @@ func TestExecutor_Execute_TopN(t *testing.T) {
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row="foo", col="a")
SetBit(field=f, row="foo", col="b")
SetBit(field=f, row="foo", col="c")
SetBit(field=f, row="foo", col="d")
SetBit(field=f, row="foo", col="e")
SetBit(field=f, row="bar", col="a")
SetBit(field=f, row="bar", col="b")
SetBit(field=f, row="baz", col="b")
SetBit(field=other, row="foo", col="a")
Set("a", f="foo")
Set("b", f="foo")
Set("c", f="foo")
Set("d", f="foo")
Set("e", f="foo")
Set("a", f="bar")
Set("b", f="bar")
Set("b", f="baz")
Set("a", other="foo")
`), nil, nil); err != nil {
t.Fatal(err)
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if diff := cmp.Diff(result, []interface{}{
[]pilosa.Pair{
@ -606,7 +596,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 4},
@ -640,7 +630,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -675,7 +665,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(other=100), n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
@ -699,7 +689,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -722,7 +712,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -755,20 +745,20 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=x, row=0, col=0)
SetBit(field=x, row=0, col=3)
SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(field=x, row=1, col=1)
SetBit(field=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`)
Set(0, x=0)
Set(3, x=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, x=0)
Set(1, x=1)
Set(`+strconv.Itoa(SliceWidth+2)+`, x=2)
SetValue(f=20, col=0)
SetValue(f=-5, col=1)
SetValue(f=-5, col=2)
SetValue(f=10, col=3)
SetValue(f=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(col=0, f=20)
SetValue(col=1, f=-5)
SetValue(col=2, f=-5)
SetValue(col=3, f=10)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, f=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, f=40)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, f=50)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, f=60)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -780,9 +770,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
cnt int64
}{
{filter: ``, exp: -5, cnt: 2},
{filter: `Bitmap(field=x, row=0)`, exp: 10, cnt: 1},
{filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1},
{filter: `Row(x=0)`, exp: 10, cnt: 1},
{filter: `Row(x=1)`, exp: -5, cnt: 1},
{filter: `Row(x=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
@ -806,9 +796,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
cnt int64
}{
{filter: ``, exp: 60, cnt: 1},
{filter: `Bitmap(field=x, row=0)`, exp: 60, cnt: 1},
{filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1},
{filter: `Row(x=0)`, exp: 60, cnt: 1},
{filter: `Row(x=1)`, exp: -5, cnt: 1},
{filter: `Row(x=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
@ -866,16 +856,16 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=x, row=0, col=0)
SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
Set(0, x=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, x=0)
SetValue(foo=20, col=0)
SetValue(bar=2000, col=0)
SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(other=1000, col=0)
SetValue(col=0, foo=20)
SetValue(col=0, bar=2000)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=40)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=50)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60)
SetValue(col=0, other=1000)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -889,7 +879,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(field=x, row=0), field=foo)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Row(x=0), field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -898,7 +888,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
// Ensure a range query can be executed.
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
func TestExecutor_Execute_Range(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
@ -915,23 +905,24 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
}
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(field=f, row=1, col=3, timestamp="2000-01-01T00:00")
SetBit(field=f, row=1, col=4, timestamp="2000-01-02T00:00")
SetBit(field=f, row=1, col=5, timestamp="2000-02-01T00:00")
SetBit(field=f, row=1, col=6, timestamp="2001-01-01T00:00")
SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00")
cc := test.MustParse(`
Set(2, f=1, 1999-12-31T00:00)
Set(3, f=1, 2000-01-01T00:00)
Set(4, f=1, 2000-01-02T00:00)
Set(5, f=1, 2000-02-01T00:00)
Set(6, f=1, 2001-01-01T00:00)
Set(7, f=1, 2002-01-01T02:00)
SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00")
SetBit(field=f, row=1, col=2, timestamp="2002-02-01T00:00")
SetBit(field=f, row=10, col=2, timestamp="2001-01-01T00:00")
`), nil, nil); err != nil {
Set(2, f=1, 1999-12-30T00:00)
Set(2, f=1, 2002-02-01T00:00)
Set(2, f=10, 2001-01-01T00:00)
`)
if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -940,7 +931,7 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
}
// Ensure a Range(bsiGroup) query can be executed.
func TestExecutor_Execute_Range(t *testing.T) {
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
@ -987,18 +978,18 @@ func TestExecutor_Execute_Range(t *testing.T) {
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
Set(0, f=0)
Set(`+strconv.Itoa(SliceWidth+1)+`, f=0)
SetValue(foo=20, col=50)
SetValue(bar=2000, col=50)
SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(other=1000, col=0)
SetValue(edge=100, col=0)
SetValue(edge=-100, col=1)
SetValue(col=50, foo=20)
SetValue(col=50, bar=2000)
SetValue(col=`+strconv.Itoa(SliceWidth)+`, foo=30)
SetValue(col=`+strconv.Itoa(SliceWidth+2)+`, foo=10)
SetValue(col=`+strconv.Itoa((5*SliceWidth)+100)+`, foo=20)
SetValue(col=`+strconv.Itoa(SliceWidth+1)+`, foo=60)
SetValue(col=0, other=1000)
SetValue(col=0, edge=100)
SetValue(col=1, edge=-100)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -1066,7 +1057,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
})
t.Run("BETWEEN", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [1, 1000])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(0 < other < 1000)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -1075,7 +1066,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Ensure that the NotNull code path gets run.
t.Run("NotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [0, 1000])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(-1 < other < 1000)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -1123,6 +1114,8 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Ensure a remote query can return a row.
func TestExecutor_Execute_Remote_Row(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
@ -1139,7 +1132,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Bitmap(field="f", row=10)` {
} else if query.String() != `Row(f=10)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{1}) {
t.Fatalf("unexpected slices: %+v", slices)
@ -1162,7 +1155,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
hldr.SetBit("i", "f", 10, SliceWidth+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -1171,6 +1164,8 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
// Ensure a remote query can return a count.
func TestExecutor_Execute_Remote_Count(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
@ -1197,7 +1192,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
hldr.SetBit("i", "f", 10, (2*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
@ -1206,6 +1201,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
@ -1225,7 +1222,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(col=2, field="f", row=10)` {
} else if query.String() != `Set(_col=2, f=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1243,7 +1240,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2)`), nil, nil); err != nil {
cc := test.MustParse("Set(2, f=10)")
if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil {
t.Fatal(err)
}
@ -1258,6 +1256,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
@ -1277,7 +1277,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(col=2, field="f", row=10, timestamp="2016-12-11T10:09")` {
} else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1297,7 +1297,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
cc := test.MustParse(`Set(2, f=10, 2016-12-11T10:09)`)
if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil {
t.Fatal(err)
}
@ -1312,6 +1313,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Ensure a remote query can return a top-n query.
func TestExecutor_Execute_Remote_TopN(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
@ -1338,11 +1341,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// slices and a second time to get the counts for a set of bitmaps.
switch remoteExecN {
case 0:
if query.String() != `TopN(field="f", n=3)` {
if query.String() != `TopN(_field="f", n=3)` {
t.Fatalf("unexpected query(0): %s", query.String())
}
case 1:
if query.String() != `TopN(field="f", ids=[0,10,30], n=3)` {
if query.String() != `TopN(_field="f", ids=[0,10,30], n=3)` {
t.Fatalf("unexpected query(1): %s", query.String())
}
default:
@ -1366,7 +1369,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
hldr.SetBit("i", "f", 30, (4*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -1377,6 +1380,56 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
}
}
// Ensure a remote query can set RowAttrs
func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) {
t.Skip("test.NewServer broken")
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.Host())
if err != nil {
t.Fatal(err)
}
c.Nodes[1].URI = *uri
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` {
t.Fatalf("unexpected query: %s", query.String())
}
return []interface{}{}, nil
}
// Create local executor data.
// The local node owns slice 1.
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
f := hldr.Field("i", "f")
s.Handler.API.Holder = hldr.Holder
hldr.SetBit("i", "f", 10, SliceWidth+1)
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
} else if m, err := f.RowAttrStore().Attrs(10); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"bat": true, "baz": int64(123)}) {
t.Fatalf("unexpected bitmap attr: %#v", m)
}
}
// Ensure executor returns an error if too many writes are in a single request.
func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
hldr := test.MustOpenHolder()
@ -1384,13 +1437,13 @@ func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.MaxWritesPerRequest = 3
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Set() Clear() Set() Set()`), nil, nil); err != pilosa.ErrTooManyWrites {
t.Fatalf("unexpected error: %s", err)
}
}
// Ensure SetColumnAttrs doesn't save `field` as an attribute
func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
@ -1401,11 +1454,11 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
// SetColumnAttrs call should exclude the field attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil)
_, err := e.Execute(context.Background(), "i", test.MustParse("Set(10, f=1)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(field='f', col=10, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(10, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}
@ -1418,11 +1471,11 @@ func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
}
// SetColumnAttrs call should not break if field is not specified
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=20)"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("Set(20, f=10)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(col=20, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(20, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}

4
fragment.go
View file

@ -1873,11 +1873,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Only sync the standard block.
for j := 0; j < len(set.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.slice*SliceWidth)+set.columnIDs[j], f.field, set.rowIDs[j])
count++
}
for j := 0; j < len(clear.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.slice*SliceWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j])
count++
}

17
handler.go
View file

@ -2,7 +2,6 @@ package pilosa
import (
"encoding/json"
"net"
)
// QueryRequest represent a request to process a query.
@ -61,18 +60,18 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
}
type Handler interface {
Serve(ln net.Listener, closing <-chan struct{})
GetAPI() *API
Serve() error
Close() error
}
type NopHandler struct{}
type nopHandler struct{}
func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {}
func (n *NopHandler) GetAPI() *API {
func (n nopHandler) Serve() error {
return nil
}
func NewNopHandler() Handler {
return &NopHandler{}
func (n nopHandler) Close() error {
return nil
}
var NopHandler Handler = nopHandler{}

2
holder_test.go
View file

@ -350,6 +350,8 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
t.Skip() // Until test.NewServer() works
s := test.NewServer()
defer s.Close()

49
http/client_test.go
View file

@ -52,6 +52,8 @@ func init() {
// Test distributed TopN Row count across 3 nodes.
func TestClient_MultiNode(t *testing.T) {
t.Skip() // Until test.NewServer() works
cluster := test.NewCluster(3)
s, hldr := createCluster(cluster)
@ -155,7 +157,7 @@ func TestClient_MultiNode(t *testing.T) {
topN := 4
queryRequest := &internal.QueryRequest{
Query: fmt.Sprintf(`TopN(field="%s", n=%d)`, "f", topN),
Query: fmt.Sprintf(`TopN(f, n=%d)`, topN),
Remote: false,
}
result, err := client[0].Query(context.Background(), "i", queryRequest)
@ -217,21 +219,17 @@ func TestClient_MultiNode(t *testing.T) {
// Ensure client can bulk import data.
func TestClient_Import(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
host := cmd.Server.Addr().String()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
// Load bitmap into cache to ensure cache gets updated.
hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created.
hldr.Row("i", "f", 0)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := MustNewClient(s.Host(), defaultClient)
c := MustNewClient(host, defaultClient)
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
@ -251,11 +249,12 @@ func TestClient_Import(t *testing.T) {
// Ensure client can bulk import value data.
func TestClient_ImportValue(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
host := cmd.Server.Addr().String()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
fldName := "f"
fo := pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -100,
@ -269,14 +268,8 @@ func TestClient_ImportValue(t *testing.T) {
t.Fatal(err)
}
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := MustNewClient(s.Host(), defaultClient)
c := MustNewClient(host, defaultClient)
if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{
{ColumnID: 1, Value: -10},
{ColumnID: 2, Value: 20},
@ -328,24 +321,16 @@ func TestClient_ImportValue(t *testing.T) {
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
// Set two bits on blocks 0 & 3.
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
hldr.SetBit("i", "f", 0, 1)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Retrieve blocks.
c := MustNewClient(s.Host(), defaultClient)
c := MustNewClient(cmd.Server.Addr().String(), defaultClient)
blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0)
if err != nil {
t.Fatal(err)

39
http/handler.go
View file

@ -15,6 +15,7 @@
package http
import (
"context"
"crypto/tls"
"encoding/json"
"expvar"
@ -53,6 +54,10 @@ type Handler struct {
API *pilosa.API
AllowedOrigins []string
ln net.Listener
server *http.Server
}
// externalPrefixFlag denotes endpoints that are intended to be exposed to clients.
@ -99,6 +104,13 @@ func OptHandlerLogger(logger pilosa.Logger) HandlerOption {
}
}
func OptHandlerListener(ln net.Listener) HandlerOption {
return func(h *Handler) error {
h.ln = ln
return nil
}
}
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler(opts ...HandlerOption) (*Handler, error) {
handler := &Handler{
@ -114,19 +126,32 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) {
}
}
if handler.API == nil {
return nil, errors.New("must pass OptHandlerAPI")
}
if handler.ln == nil {
return nil, errors.New("must pass OptHandlerListener")
}
handler.server = &http.Server{Handler: handler}
return handler, nil
}
func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) {
server := &http.Server{Handler: h}
go func() {
<-closing
server.Close()
}()
err := server.Serve(ln)
func (h *Handler) Serve() error {
err := h.server.Serve(h.ln)
if err != nil && err.Error() != "http: Server closed" {
h.Logger.Printf("HTTP handler terminated with error: %s\n", err)
return errors.Wrap(err, "serve http")
}
return nil
}
func (h *Handler) Close() error {
// TODO: timeout?
err := h.server.Shutdown(context.Background())
return errors.Wrap(err, "shutdown http server")
}
func (h *Handler) populateValidators() {

948
http/handler_test.go
View file

@ -15,952 +15,28 @@
package http_test
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
"net/http/httptest"
"reflect"
"strings"
"net"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
func TestHandlerPanics(t *testing.T) {
h := test.MustNewHandler()
bufLogger := test.NewBufferLogger()
h.Handler.Logger = bufLogger
w := httptest.NewRecorder()
// will panic since Handler has no Holder set up
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil))
bufbytes, err := bufLogger.ReadAll()
if err != nil {
t.Fatalf("reading all logoutput: %v", err)
func TestHandlerOptions(t *testing.T) {
_, err := http.NewHandler()
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) {
t.Fatalf("expected panic in log, but got: %s", bufbytes)
_, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{}))
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
if w.Code != gohttp.StatusInternalServerError {
t.Fatalf("expected internal server error, but got: %v", w.Code)
}
bodyBytes := w.Body.Bytes()
if !bytes.Contains(bodyBytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) {
t.Fatalf("response to client should have panic, but got %s", bodyBytes)
}
}
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_NotFound(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != gohttp.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler can return the schema.
func TestHandler_Schema(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return the status.
func TestHandler_Status(t *testing.T) {
s := test.NewServer()
hldr := test.MustOpenHolder()
defer s.Close()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateNormal)
h.API.StatusHandler = s
s.Handler = h
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
func TestHandler_Info(t *testing.T) {
s := test.NewServer()
defer s.Close()
h := test.MustNewHandler()
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can abort a cluster resize.
func TestHandler_ClusterResizeAbort(t *testing.T) {
t.Run("No resize job", func(t *testing.T) {
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateResizing)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != gohttp.StatusOK {
bod, err := ioutil.ReadAll(w.Body)
t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err)
} else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
}
// Ensure the handler can return the maxslice map.
func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8)
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Count(Bitmap(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Count(Bitmap(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return []interface{}{uint64(100)}, nil
}
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
Query: "Count(Bitmap(id=100))",
Slices: []uint64{0, 1},
})
ln, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler returns an error when parsing bad arguments.
func TestHandler_Query_Args_Err(t *testing.T) {
w := httptest.NewRecorder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
func TestHandler_Query_Params_Err(t *testing.T) {
w := httptest.NewRecorder()
test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 100 {
t.Fatalf("unexpected n: %d", n)
}
}
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return []interface{}{r}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can execute a query that returns a row with column attributes as JSON.
func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) {
hldr := test.NewHolder()
defer hldr.Close()
// Create index and set column attributes.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return []interface{}{r}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can execute a query that returns a row as protobuf.
func TestHandler_Query_Row_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return []interface{}{r}, nil
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
}
// Ensure the handler can execute a query that returns a row with column attributes as protobuf.
func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) {
hldr := test.NewHolder()
defer hldr.Close()
// Create index and set column attributes.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return []interface{}{r}, nil
}
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
Query: "Bitmap(id=100)",
ColumnAttrs: true,
})
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
if a := resp.ColumnAttrSets; len(a) != 1 {
t.Fatalf("unexpected column attributes length: %d", len(a))
} else if a[0].ID != 1 {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
}
}
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
{ID: 3, Count: 4},
}}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
{ID: 3, Count: 4},
}}, nil
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
}
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.Err; s != `executing: marker` {
t.Fatalf("unexpected error: %s", s)
}
}
// Ensure the handler returns "method not allowed" for non-POST queries.
func TestHandler_Query_MethodNotAllowed(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil))
if w.Code != gohttp.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler returns an error if there is a parsing error..
func TestHandler_Query_ErrParse(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can delete an index.
func TestHandler_Index_Delete(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Create index.
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
// Send request to delete index.
resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Verify body response.
if resp.StatusCode != gohttp.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
} else if buf, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if string(buf) != "{}\n" {
t.Fatalf("unexpected response body: %s", buf)
}
// Verify index is gone.
if hldr.Index("i") != nil {
t.Fatal("expected nil index")
}
}
// Ensure handler can delete a field.
func TestHandler_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i0").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
}
// Ensure the handler can return data in differing blocks for an index.
func TestHandler_Index_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
// Retrieve block checksums.
blks, err := index.ColumnAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
// Remove block #0 and alter block 2's checksum.
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req, err := gohttp.NewRequest(
"POST",
s.URL+"/index/i/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
client := &gohttp.Client{}
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Read and validate body.
if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return data in differing blocks for a field.
func TestHandler_Field_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
}
if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
// Retrieve block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
// Remove block #0 and alter block 2's checksum.
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req, err := gohttp.NewRequest(
"POST",
s.URL+"/index/i/field/meta/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
client := &gohttp.Client{}
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Read and validate body.
if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
version := pilosa.Version
if strings.HasPrefix(version, "v") {
version = version[1:]
}
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"`+version+`"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// Ensure the handler can return a list of nodes for a fragment.
func TestHandler_Fragment_Nodes(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(3)
h.API.Cluster.ReplicaN = 2
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
// invalid argument should return BadRequest
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
// index is required
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler can return expvars without panicking.
func TestHandler_Expvars(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
func MustReadAll(r io.Reader) []byte {
buf, err := ioutil.ReadAll(r)
if err != nil {
panic(err)
}
return buf
}
func TestHandler_RecalculateCaches(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
if w.Code != gohttp.StatusNoContent {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
func TestHandler_CORS(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// No CORS config present, so should fail
handler := test.MustNewHandler()
req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil)
req.Header.Add("Origin", "http://test/")
req.Header.Add("Access-Control-Request-Method", "POST")
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
result := w.Result()
// This handler does not support CORS, return Method Not Allowed (405)
if result.StatusCode != 405 {
t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode)
}
// CORS config should allow preflight response
handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"}))
w = httptest.NewRecorder()
handler.ServeHTTP(w, req)
result = w.Result()
if result.StatusCode != 200 {
t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode)
}
if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" {
t.Fatal("CORS header not present")
_, err = http.NewHandler(http.OptHandlerListener(ln))
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
}

2
http/translator_test.go
View file

@ -15,6 +15,8 @@ import (
)
func TestTranslateStore_Reader(t *testing.T) {
t.Skip() // Until test.NewServer() works
// Ensure client can connect and stream the translate store data.
t.Run("OK", func(t *testing.T) {
t.Run("ServerDisconnect", func(t *testing.T) {

194
pql/ast.go
View file

@ -26,6 +26,186 @@ import (
// Query represents a PQL query.
type Query struct {
Calls []*Call
lastField string
lastCond Token
inList bool
callStack []*Call
conditional []string
}
func (q *Query) startCall(name string) {
newCall := &Call{Name: name}
q.callStack = append(q.callStack, newCall)
if len(q.callStack) == 1 {
q.Calls = append(q.Calls, newCall)
} else {
calls := q.callStack[len(q.callStack)-2].Children
q.callStack[len(q.callStack)-2].Children = append(calls, newCall)
}
}
func (q *Query) endCall() {
q.callStack = q.callStack[:len(q.callStack)-1]
}
func (q *Query) addPosNum(key, value string) {
q.addField(key)
q.addNumVal(value)
}
func (q *Query) addPosStr(key, value string) {
q.addField(key)
q.addVal(value)
}
func (q *Query) startConditional() {
q.conditional = make([]string, 0)
call := q.callStack[len(q.callStack)-1]
if call.Args == nil {
call.Args = make(map[string]interface{})
}
}
func (q *Query) condAdd(val string) {
q.conditional = append(q.conditional, val)
}
func (q *Query) endConditional() {
// do stuff
if len(q.conditional) != 5 {
panic(fmt.Sprintf("conditional of wrong length: %#v", q.conditional))
}
low, _ := strconv.ParseInt(q.conditional[0], 10, 64)
field := q.conditional[2]
high, _ := strconv.ParseInt(q.conditional[4], 10, 64)
if q.conditional[1] == "<" {
low++
}
if q.conditional[3] == "<=" {
high++
}
call := q.callStack[len(q.callStack)-1]
call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}}
q.conditional = nil
}
func (q *Query) addField(field string) {
if q.lastField != "" {
panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField))
}
q.lastField = field
call := q.callStack[len(q.callStack)-1]
if call.Args == nil {
call.Args = make(map[string]interface{})
}
}
func (q *Query) addVal(val interface{}) {
if q.lastField == "" {
panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val))
}
call := q.callStack[len(q.callStack)-1]
if q.inList {
list := call.Args[q.lastField].([]interface{})
call.Args[q.lastField] = append(list, val)
return
}
if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: val,
}
} else {
call.Args[q.lastField] = val
}
q.lastField = ""
q.lastCond = ILLEGAL
}
func (q *Query) addNumVal(val string) {
if q.lastField == "" {
panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val))
}
var ival interface{}
var err error
if strings.Contains(val, ".") {
ival, err = strconv.ParseFloat(val, 64)
} else {
ival, err = strconv.ParseInt(val, 10, 64)
}
if err != nil {
panic(err)
}
call := q.callStack[len(q.callStack)-1]
if q.inList {
if q.lastCond != ILLEGAL {
list := call.Args[q.lastField].(*Condition).Value.([]interface{})
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: append(list, ival),
}
} else {
list := call.Args[q.lastField].([]interface{})
call.Args[q.lastField] = append(list, ival)
}
return
} else if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: ival,
}
} else {
call.Args[q.lastField] = ival
}
q.lastField = ""
q.lastCond = ILLEGAL
}
func (q *Query) startList() {
call := q.callStack[len(q.callStack)-1]
if q.lastCond != ILLEGAL {
call.Args[q.lastField] = &Condition{
Op: q.lastCond,
Value: make([]interface{}, 0),
}
} else {
call.Args[q.lastField] = make([]interface{}, 0)
}
q.inList = true
}
func (q *Query) endList() {
q.inList = false
q.lastField = ""
q.lastCond = ILLEGAL
}
func (q *Query) addGT() {
q.lastCond = GT
}
func (q *Query) addLT() {
q.lastCond = LT
}
func (q *Query) addGTE() {
q.lastCond = GTE
}
func (q *Query) addLTE() {
q.lastCond = LTE
}
func (q *Query) addEQ() {
q.lastCond = EQ
}
func (q *Query) addNEQ() {
q.lastCond = NEQ
}
func (q *Query) addBTWN() {
q.lastCond = BETWEEN
}
// WriteCallN returns the number of mutating calls.
@ -33,7 +213,7 @@ func (q *Query) WriteCallN() int {
var n int
for _, call := range q.Calls {
switch call.Name {
case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs":
case "Set", "Clear", "SetRowAttrs", "SetColumnAttrs":
n++
}
}
@ -73,6 +253,18 @@ type Call struct {
Children []*Call
}
// FieldArg determines which key-value pair contains the field and rowID,
// in the case of arguments like Set(colID, field=rowID).
// Returns the field as a string if present, or an error if not.
func (c *Call) FieldArg() (string, error) {
for arg := range c.Args {
if !strings.HasPrefix(arg, "_") {
return arg, nil
}
}
return "", fmt.Errorf("No field argument specified")
}
// UintArg is for reading the value at key from call.Args as a uint64. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. The value is assumed to be a uint64 or an int64 and

299
pql/parser.go
View file

@ -15,10 +15,11 @@
package pql
import (
"fmt"
"io"
"strconv"
"io/ioutil"
"strings"
"github.com/pkg/errors"
)
// TimeFormat is the go-style time format used to parse string dates.
@ -26,13 +27,16 @@ const TimeFormat = "2006-01-02T15:04"
// Parser represents a parser for the PQL language.
type Parser struct {
scanner *bufScanner
r io.Reader
//scanner *bufScanner
PQL
}
// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
return &Parser{
scanner: newBufScanner(r),
r: r,
// scanner: newBufScanner(r),
}
}
@ -43,287 +47,18 @@ func ParseString(s string) (*Query, error) {
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
q := &Query{}
for {
call, err := p.parseCall()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
q.Calls = append(q.Calls, call)
}
// Require at least one call.
if len(q.Calls) == 0 {
return nil, io.ErrUnexpectedEOF
}
return q, nil
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (*Call, error) {
var c Call
// Read call name.
tok, pos, lit := p.scanIgnoreWhitespace()
if tok == EOF {
return nil, io.EOF
} else if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}
c.Name = lit
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse children first.
children, err := p.parseChildren()
buf, err := ioutil.ReadAll(p.r)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "reading buffer to parse")
}
c.Children = children
// If next token is a closing paren then exit.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
return &c, nil
} else if tok == IDENT {
p.unscan(1)
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit)
p.PQL = PQL{
Buffer: string(buf),
}
// Parse key/value arguments.
args, err := p.parseArgs()
p.Init()
err = p.PQL.Parse()
if err != nil {
return nil, err
}
c.Args = args
// Scan closing parenthesis.
if err := p.expect(RPAREN); err != nil {
return nil, err
}
return &c, nil
}
// parseChildren parses call children.
func (p *Parser) parseChildren() ([]*Call, error) {
var offset int
var children []*Call
for {
// Ensure next two tokens are IDENT+LPAREN.
if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT {
p.unscanIgnoreWhitespace(1 + offset)
return children, nil
}
if tok, _, _ := p.scan(); tok != LPAREN {
p.unscanIgnoreWhitespace(2 + offset)
return children, nil
}
// Push tokens back on scanner and parse as a call.
p.unscan(2)
child, err := p.parseCall()
if err != nil {
return nil, err
}
children = append(children, child)
// Exit if closing paren.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
p.unscan(1)
return children, nil
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
}
// Make sure comma is unscanned.
offset = 1
}
}
// parseArgs parses key/value arguments.
func (p *Parser) parseArgs() (map[string]interface{}, error) {
args := make(map[string]interface{})
for {
// Parse key.
tok, pos, lit := p.scanIgnoreWhitespace()
if tok == RPAREN {
p.unscan(1)
return args, nil
} else if tok != IDENT {
return nil, parseErrorf(pos, "expected argument key, found %q", lit)
}
key := lit
// Expect '=' or a comparison next.
var op Token
switch tok, pos, lit := p.scanIgnoreWhitespace(); tok {
case ASSIGN:
case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN:
op = tok
default:
return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit)
}
// Parse value.
var value interface{}
tok, pos, lit = p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
value = true
} else if lit == "false" {
value = false
} else if lit == "null" {
value = nil
} else {
value = lit
}
case STRING:
value = lit
case INTEGER:
v, err := strconv.ParseInt(lit, 10, 64)
if err != nil {
return nil, err
}
value = v
case FLOAT:
v, err := strconv.ParseFloat(lit, 64)
if err != nil {
return nil, err
}
value = v
case LBRACK:
v, err := p.parseList()
if err != nil {
return nil, err
}
value = v
default:
return nil, parseErrorf(pos, "invalid argument value: %q", lit)
}
// Ensure key doesn't already exist.
if _, ok := args[key]; ok {
return nil, parseErrorf(pos, "argument key already used: %s", key)
}
// If op is specified then create a condition.
if op != 0 {
value = &Condition{Op: op, Value: value}
}
// Add key/value pair to arguments.
args[key] = value
// Exit if closing paren.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
p.unscan(1)
return args, nil
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
}
}
}
// parseList parses a list of primitives. This is used by the TopN() filters.
func (p *Parser) parseList() ([]interface{}, error) {
var values []interface{}
for {
// Read next value.
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
if lit == "true" {
values = append(values, true)
} else if lit == "false" {
values = append(values, false)
} else {
values = append(values, lit)
}
case STRING:
values = append(values, lit)
case INTEGER:
v, err := strconv.ParseInt(lit, 10, 64)
if err != nil {
return nil, err
}
values = append(values, v)
default:
return nil, parseErrorf(pos, "invalid list value: %q", lit)
}
// Expect a comma or closing bracket next.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected comma, found %q", lit)
}
}
return values, nil
}
// scan returns the next token from the scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
// scanIgnoreWhitespace returns the next non-whitespace token from the scanner.
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
tok, pos, lit = p.scan()
if tok == WS {
tok, pos, lit = p.scan()
}
return
}
// unscan returns the last n tokens back to the scanner.
func (p *Parser) unscan(n int) {
for i := 0; i < n; i++ {
p.scanner.unscan()
}
}
// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner.
func (p *Parser) unscanIgnoreWhitespace(n int) {
for i := 0; i < n; {
p.scanner.unscan()
if tok, _, _ := p.scanner.curr(); tok != WS {
i++
}
}
}
// expect returns an error if the next token is not exp.
func (p *Parser) expect(exp Token) error {
if tok, pos, lit := p.scan(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// pos returns the current position.
func (p *Parser) pos() Pos { return p.scanner.pos() }
// ParseError represents an error that occurred while parsing a PQL query.
type ParseError struct {
Message string
Pos Pos
}
// Error returns a string representation of e.
func (e *ParseError) Error() string {
return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1)
}
// parseErrorf returns a formatted parse error.
func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
return &ParseError{
Message: fmt.Sprintf(format, args...),
Pos: pos,
return nil, errors.Wrap(err, "parsing")
}
p.Execute()
return &p.Query, nil
}

10
pql/parser_test.go
View file

@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) {
// Parse with both child calls and arguments.
t.Run("ChildrenAndArguments", func(t *testing.T) {
q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`)
q, err := pql.ParseString(`TopN(f, Bitmap(id=100, field=other), n=3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
@ -145,7 +145,7 @@ func TestParser_Parse(t *testing.T) {
Name: "Bitmap",
Args: map[string]interface{}{"id": int64(100), "field": "other"},
}},
Args: map[string]interface{}{"n": int64(3), "field": "f"},
Args: map[string]interface{}{"n": int64(3), "_field": "f"},
},
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
@ -154,15 +154,15 @@ func TestParser_Parse(t *testing.T) {
// Parse a list argument.
t.Run("ListArgument", func(t *testing.T) {
q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`)
q, err := pql.ParseString(`TopN(f, ids=[0,10,30])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "TopN",
Args: map[string]interface{}{
"field": "f",
"ids": []interface{}{int64(0), int64(10), int64(30)},
"_field": "f",
"ids": []interface{}{int64(0), int64(10), int64(30)},
},
},
) {

75
pql/pql.peg Normal file
View file

@ -0,0 +1,75 @@
package pql
type PQL Peg {
Query
}
Calls <- whitesp (Call whitesp)* !.
Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()}
/ 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()}
/ 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
/ 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
/ 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()}
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
allargs <- Call (comma Call)* (comma args)? / args / sp
args <- arg (comma args)? sp
arg <- ( field sp '=' sp value
/ field sp COND sp value
)
COND <- ( '><' { p.addBTWN() }
/ '<=' { p.addLTE() }
/ '>=' { p.addGTE() }
/ '==' { p.addEQ() }
/ '!=' { p.addNEQ() }
/ '<' { p.addLT() }
/ '>' { p.addGT() }
)
conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])}
condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])}
condfield <- <fieldExpr> sp {p.condAdd(buffer[begin:end])}
timerange <- field sp '=' sp value comma <timestampfmt> {p.addPosStr("_start", buffer[begin:end])} comma <timestampfmt> {p.addPosStr("_end", buffer[begin:end])}
value <- ( item
/ lbrack { p.startList() } list rbrack { p.endList() }
)
list <- item (comma list)?
item <- ( 'null' &(comma / sp close) { p.addVal(nil) }
/ 'true' &(comma / sp close) { p.addVal(true) }
/ 'false' &(comma / sp close) { p.addVal(false) }
/ < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) }
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) }
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
/ '"' < doublequotedstring > '"' { p.addVal(buffer[begin:end]) }
/ '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) }
)
doublequotedstring <- ( [^"\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )*
singlequotedstring <- ( [^'\\\n] / '\\n' / '\\\"' / '\\\'' / '\\\\' )*
fieldExpr <- [[A-Z]] ( [[A-Z]] / [0-9] / '_' )*
field <- <fieldExpr / reserved> { p.addField(buffer[begin:end]) }
reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field')
posfield <- <fieldExpr> { p.addPosStr("_field", buffer[begin:end]) }
uint <- [1-9] [0-9]* / '0'
uintrow <- <uint>{p.addPosNum("_row", buffer[begin:end])}
col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
/ '"' <doublequotedstring> '"' {p.addPosStr("_col", buffer[begin:end])}
)
open <- '(' sp
close <- ')' sp
sp <- ( ' ' / '\t' )*
comma <- sp ',' whitesp
lbrack <- '[' sp
rbrack <- sp ']' sp
whitesp <- ( ' ' / '\t' / '\n' )*
IDENT <- [[A-Z]] ([[A-Z]] / [0-9])*
timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9]
timestampfmt <- '"' timestampbasicfmt '"' / '\'' timestampbasicfmt '\'' / timestampbasicfmt
timestamp <- <timestampfmt> {p.addPosStr("_timestamp", buffer[begin:end])}

2843
pql/pql.peg.go Normal file
View file

File diff suppressed because it is too large Load diff

526
pql/pqlpeg_test.go Normal file
View file

@ -0,0 +1,526 @@
package pql
import (
"reflect"
"strconv"
"testing"
)
func TestPEG(t *testing.T) {
p := PQL{Buffer: `
SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]}
p.Init()
err := p.Parse()
if err != nil {
t.Fatalf("parse error: %v", err)
}
p.Execute()
p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`}
p.Init()
err = p.Parse()
if err == nil {
t.Fatalf("should have been an error because of the interior unescaped double quote")
}
q, err := ParseString("TopN(blah, Bitmap(id==other), field=f, n=0)")
if err != nil {
t.Fatalf("should have parsed: %v", err)
}
if q.String() != `TopN(Bitmap(id == "other"), _field="blah", field="f", n=0)` {
t.Fatalf("Failed, got: %s", q)
}
q, err = ParseString("C(a=falsen0)")
if err != nil {
t.Fatalf("falsen0 should have been parsed as a string")
}
q, err = ParseString("Bitmap(row=4, did==other)")
if err != nil {
t.Fatalf("should have parsed: %v", err)
}
if q.String() != `Bitmap(did == "other", row=4)` {
t.Fatalf("got %s", q)
}
}
func TestOldPQL(t *testing.T) {
_, err := ParseString(`SetBit(f=11, col=1)`)
if err != nil {
t.Fatalf("should have parsed: %v", err)
}
}
func TestPEGWorking(t *testing.T) {
tests := []struct {
name string
input string
ncalls int
}{
{
name: "Empty",
input: "",
ncalls: 0},
{
name: "Set",
input: "Set(2, f=10)",
ncalls: 1},
{
name: "SetTime",
input: "Set(2, f=1, 1999-12-31T00:00)",
ncalls: 1},
{
name: "DoubleSet",
input: "Set(1, a=4)Set(2, a=4)",
ncalls: 2},
{
name: "DoubleSetSpc",
input: "Set(1, a=4) Set(2, a=4)",
ncalls: 2},
{
name: "DoubleSetNewline",
input: "Set(1, a=4) \n Set(2, a=4)",
ncalls: 2},
{
name: "SetWithArbCall",
input: "Set(1, a=4)Blerg(z=ha)",
ncalls: 2},
{
name: "SetArbSet",
input: "Set(1, a=4)Blerg(z=ha)Set(2, z=99)",
ncalls: 3},
{
name: "ArbSetArb",
input: "Arb(q=1, a=4)Set(1, z=9)Arb(z=99)",
ncalls: 3},
{
name: "SetStringArg",
input: "Set(1, a=zoom)",
ncalls: 1},
{
name: "SetManyArgs",
input: "Set(1, a=4, b=5)",
ncalls: 1},
{
name: "SetManyMixedArgs",
input: "Set(1, a=4, bsd=haha)",
ncalls: 1},
{
name: "SetTimestamp",
input: "Set(1, a=4, 2017-04-03T19:34)",
ncalls: 1},
{
name: "Union()",
input: "Union()",
ncalls: 1},
{
name: "UnionOneRow",
input: "Union(Row(a=1))",
ncalls: 1},
{
name: "UnionTwoRows",
input: "Union(Row(a=1), Row(z=44))",
ncalls: 1},
{
name: "UnionNested",
input: "Union(Intersect(Row(), Union(Row(), Row())), Row())",
ncalls: 1},
{
name: "TopN no args",
input: "TopN(boondoggle)",
ncalls: 1},
{
name: "TopN with args",
input: "TopN(boon, doggle=9)",
ncalls: 1},
{
name: "double quoted args",
input: `B(a="zm''e")`,
ncalls: 1},
{
name: "single quoted args",
input: `B(a='zm""e')`,
ncalls: 1},
{
name: "SetRowAttrs",
input: "SetRowAttrs(blah, 9, a=47)",
ncalls: 1},
{
name: "SetRowAttrs2args",
input: "SetRowAttrs(blah, 9, a=47, b=bval)",
ncalls: 1},
{
name: "SetColumnAttrs",
input: "SetColumnAttrs(9, a=47)",
ncalls: 1},
{
name: "SetColumnAttrs2args",
input: "SetColumnAttrs(9, a=47, b=bval)",
ncalls: 1},
{
name: "Clear",
input: "Clear(1, a=53)",
ncalls: 1},
{
name: "Clear2args",
input: "Clear(1, a=53, b=33)",
ncalls: 1},
{
name: "TopN",
input: "TopN(myfield, n=44)",
ncalls: 1},
{
name: "TopNBitmap",
input: "TopN(myfield, Row(a=47), n=10)",
ncalls: 1},
{
name: "RangeLT",
input: "Range(a < 4)",
ncalls: 1},
{
name: "RangeGT",
input: "Range(a > 4)",
ncalls: 1},
{
name: "RangeLTE",
input: "Range(a <= 4)",
ncalls: 1},
{
name: "RangeGTE",
input: "Range(a >= 4)",
ncalls: 1},
{
name: "RangeEQ",
input: "Range(a == 4)",
ncalls: 1},
{
name: "RangeNEQ",
input: "Range(a != null)",
ncalls: 1},
{
name: "RangeLTLT",
input: "Range(4 < a < 9)",
ncalls: 1},
{
name: "RangeLTLTE",
input: "Range(4 < a <= 9)",
ncalls: 1},
{
name: "RangeLTELT",
input: "Range(4 <= a < 9)",
ncalls: 1},
{
name: "RangeLTELTE",
input: "Range(4 <= a <= 9)",
ncalls: 1},
{
name: "RangeTime",
input: "Range(a=4, 2010-07-04T00:00, 2010-08-04T00:00)",
ncalls: 1},
{
name: "RangeTimeQuotes",
input: `Range(a=4, '2010-07-04T00:00', "2010-08-04T00:00")`,
ncalls: 1},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
q, err := ParseString(test.input)
if err != nil {
t.Fatalf("parsing query '%s': %v", test.input, err)
}
if len(q.Calls) != test.ncalls {
t.Fatalf("wrong number of calls for '%s': %#v", test.input, q.Calls)
}
})
}
}
func TestPEGErrors(t *testing.T) {
tests := []struct {
name string
input string
}{
{
name: "SetNoParens",
input: "Set"},
{
name: "SetBadTimestamp",
input: "Set(1, a=4, 2017-94-03T19:34)"},
{
name: "SetTimestampNoArg",
input: "Set(1, 2017-04-03T19:34)"},
{
name: "SetStartingComma",
input: "Set(, 1, a=4)"},
{
name: "StartinCommaArb",
input: "Zeeb(, a=4)"},
{
name: "SetRowAttrs0args",
input: "SetRowAttrs(blah, 9)"},
{
name: "Clear0args",
input: "Clear(9)"},
{
name: "RangeTimeGT",
input: "Range(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"},
{
name: "RangeTimeOneStamp",
input: "Range(a=4, 2010-07-04T00:00)"},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
q, err := ParseString(test.input)
if err == nil {
t.Fatalf("parsing query '%s' - expected error, got: %s", test.input, q)
}
})
}
}
func TestPQLDeepEquality(t *testing.T) {
tests := []struct {
name string
call string
exp *Call
}{
{
name: "Set",
call: "Set(1, a=7, 2010-07-08T14:44)",
exp: &Call{
Name: "Set",
Args: map[string]interface{}{
"a": int64(7),
"_col": int64(1),
"_timestamp": "2010-07-08T14:44",
},
}},
{
name: "SetRowAttrs",
call: "SetRowAttrs(myfield, 9, z=4)",
exp: &Call{
Name: "SetRowAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_field": "myfield",
"_row": int64(9),
},
}},
{
name: "SetColumnAttrs",
call: "SetColumnAttrs(9, z=4)",
exp: &Call{
Name: "SetColumnAttrs",
Args: map[string]interface{}{
"z": int64(4),
"_col": int64(9),
},
}},
{
name: "Clear",
call: "Clear(1, a=7)",
exp: &Call{
Name: "Clear",
Args: map[string]interface{}{
"a": int64(7),
"_col": int64(1),
},
}},
{
name: "TopN",
call: "TopN(myfield, Row(), a=7)",
exp: &Call{
Name: "TopN",
Args: map[string]interface{}{
"a": int64(7),
"_field": "myfield",
},
Children: []*Call{
{Name: "Row"},
},
}},
{
name: "RangeEQ",
call: "Range(a==7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: EQ,
Value: int64(7),
},
},
}},
{
name: "RangeLT",
call: "Range(a<7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: LT,
Value: int64(7),
},
},
}},
{
name: "RangeLTE",
call: "Range(a<=7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: LTE,
Value: int64(7),
},
},
}},
{
name: "RangeGTE",
call: "Range(a>=7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: GTE,
Value: int64(7),
},
},
}},
{
name: "RangeGT",
call: "Range(a>7)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: GT,
Value: int64(7),
},
},
}},
{
name: "RangeNEQ",
call: "Range(a!=null)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: NEQ,
Value: nil,
},
},
}},
{
name: "RangeLTELT",
call: "Range(4 <= a < 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(4), int64(9)},
},
},
}},
{
name: "RangeLTLT",
call: "Range(4 < a < 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(5), int64(9)},
},
},
}},
{
name: "RangeLTELTE",
call: "Range(4 <= a <= 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(4), int64(10)},
},
},
}},
{
name: "RangeLTLTE",
call: "Range(4 < a <= 9)",
exp: &Call{
Name: "Range",
Args: map[string]interface{}{
"a": &Condition{
Op: BETWEEN,
Value: []interface{}{int64(5), int64(10)},
},
},
}},
{
name: "Sum",
call: "Sum(field=f)",
exp: &Call{
Name: "Sum",
Args: map[string]interface{}{
"field": "f",
},
}},
{
name: "SumChild",
call: "Sum(Row(), field=f)",
exp: &Call{
Name: "Sum",
Args: map[string]interface{}{
"field": "f",
},
Children: []*Call{
{Name: "Row"},
},
}},
{
name: "MinChild",
call: "Min(Row(), field=f)",
exp: &Call{
Name: "Min",
Args: map[string]interface{}{
"field": "f",
},
Children: []*Call{
{Name: "Row"},
},
}},
{
name: "MaxChild",
call: "Max(Row(), field=f)",
exp: &Call{
Name: "Max",
Args: map[string]interface{}{
"field": "f",
},
Children: []*Call{
{Name: "Row"},
},
}},
}
for i, test := range tests {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
q, err := ParseString(test.call)
if err != nil {
t.Fatalf("parsing query '%s': %v", test.call, err)
}
if !reflect.DeepEqual(test.exp, q.Calls[0]) {
t.Fatalf("unexpected call:\n%s\ninstead of:\n%s\n'%#v'\ninstead of:\n'%#v'", q.Calls[0], test.exp, q.Calls[0], test.exp)
}
})
}
}

303
pql/scanner.go
View file

@ -1,303 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pql
import (
"bufio"
"bytes"
"io"
"unicode"
)
// Scanner represents a PQL lexical scanner.
type Scanner struct {
r io.RuneScanner
pos Pos
}
// NewScanner returns a new instance of Scanner.
func NewScanner(r io.Reader) *Scanner {
return &Scanner{r: bufio.NewReader(r)}
}
// Scan returns the next token and position from the underlying reader.
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
pos = s.pos
// Read next code point.
ch := s.read()
// If we see whitespace then consume all contiguous whitespace.
// If we see a letter, or certain acceptable special characters, then consume
// as an ident or reserved word. If we see quotes, then scan as string.
if isWhitespace(ch) {
s.unread()
return s.scanWhitespace()
} else if isIdentFirstChar(ch) {
s.unread()
return s.scanIdent()
} else if isDigit(ch) || ch == '-' {
s.unread()
return s.scanNumber()
} else if ch == '"' || ch == '\'' {
s.unread()
return s.scanString()
}
// Otherwise parse individual characters.
switch ch {
case eof:
return EOF, pos, ""
case '=':
if next := s.read(); next == '=' {
return EQ, pos, "=="
}
s.unread()
return ASSIGN, pos, string(ch)
case '!':
if next := s.read(); next == '=' {
return NEQ, pos, "!="
}
s.unread()
return ASSIGN, pos, string(ch)
case '<':
if next := s.read(); next == '=' {
return LTE, pos, "<="
}
s.unread()
return LT, pos, string(ch)
case '>':
next := s.read()
if next == '=' {
return GTE, pos, ">="
} else if next == '<' {
return BETWEEN, pos, "><"
}
s.unread()
return GT, pos, string(ch)
case ',':
return COMMA, pos, string(ch)
case '(':
return LPAREN, pos, string(ch)
case ')':
return RPAREN, pos, string(ch)
case '[':
return LBRACK, pos, string(ch)
case ']':
return RBRACK, pos, string(ch)
default:
return ILLEGAL, pos, string(ch)
}
}
// read returns the next code point from the underlying reader and updates the pos.
func (s *Scanner) read() rune {
// Read next rune from underlying reader.
ch, _, err := s.r.ReadRune()
if err != nil {
return eof
}
// Update position information.
if ch == '\n' {
s.pos.Line++
s.pos.Char = 0
} else {
s.pos.Char++
}
return ch
}
// unread pushes the previously read rune back onto the reader.
func (s *Scanner) unread() {
if s.pos.Char == 0 {
s.pos.Line--
} else {
s.pos.Char--
}
s.r.UnreadRune()
}
// scanWhitespace consumes the current rune and all contiguous whitespace.
func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isWhitespace(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
return WS, pos, buf.String()
}
func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isIdentChar(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
lit = buf.String()
// If the literal matches a keyword then return that keyword.
if tok = Lookup(lit); tok != IDENT {
return tok, pos, lit
}
return IDENT, pos, lit
}
// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character.
func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) {
pos = s.pos
tok = INTEGER
var buf bytes.Buffer
var seenDot bool
first := true
for {
ch := s.read()
if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') {
s.unread()
break
}
if ch == '.' {
seenDot = true
tok = FLOAT
}
buf.WriteRune(ch)
first = false
}
return tok, pos, buf.String()
}
// scanString consumes a single-quoted or double-quoted string.
func (s *Scanner) scanString() (tok Token, pos Pos, lit string) {
pos = s.pos
// This must be either a single- or double-quote.
ending := s.read()
var buf bytes.Buffer
for {
ch := s.read()
if ch == ending {
break
} else if ch == '\n' || ch == eof {
return BADSTRING, pos, buf.String()
} else if ch == '\\' {
next := s.read()
if next == 'n' {
buf.WriteRune('\n')
} else if next == '\\' {
buf.WriteRune('\\')
} else if next == '"' {
buf.WriteRune('"')
} else if next == '\'' {
buf.WriteRune('\'')
} else {
return BADSTRING, pos, buf.String()
}
} else {
buf.WriteRune(ch)
}
}
return STRING, pos, buf.String()
}
// bufScanner represents a wrapper for scanner to add a buffer.
// It provides a fixed-length circular buffer that can be unread.
type bufScanner struct {
s *Scanner
i int // buffer index
n int // buffer size
buf [8]struct {
tok Token
pos Pos
lit string
}
}
// newBufScanner returns a new buffered scanner for a reader.
func newBufScanner(r io.Reader) *bufScanner {
return &bufScanner{s: NewScanner(r)}
}
// Scan reads the next token from the scanner.
func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) {
// If we have unread tokens then read them off the buffer first.
if s.n > 0 {
s.n--
return s.curr()
}
// Move buffer position forward and save the token.
s.i = (s.i + 1) % len(s.buf)
buf := &s.buf[s.i]
buf.tok, buf.pos, buf.lit = s.s.Scan()
return s.curr()
}
// unscan pushes the previously token back onto the buffer.
func (s *bufScanner) unscan() { s.n++ }
// curr returns the last read token.
func (s *bufScanner) curr() (tok Token, pos Pos, lit string) {
buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)]
return buf.tok, buf.pos, buf.lit
}
// pos returns the current position.
func (s *bufScanner) pos() Pos {
_, pos, _ := s.curr()
return pos
}
// isWhitespace returns true if the rune a Unicode space character.
func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) }
// isLetter returns true if the rune is a letter.
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
// isDigit returns true if the rune is a digit.
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
// isIdentChar returns true if the rune can be used in an unquoted identifier.
func isIdentChar(ch rune) bool {
return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.'
}
// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }
const eof = rune(0)

74
pql/scanner_test.go
View file

@ -1,74 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pql_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/pql"
)
func TestScanner_Scan(t *testing.T) {
var tests = []struct {
name string
s string
tok pql.Token
lit string
pos pql.Pos
}{
// Special tokens (EOF, ILLEGAL, WS)
{name: "EOF", s: ``, tok: pql.EOF},
{name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`},
{name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "},
{name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"},
{name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"},
{name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`},
{name: "EQ", s: `==`, tok: pql.EQ, lit: `==`},
{name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`},
{name: "LT", s: `<`, tok: pql.LT, lit: `<`},
{name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`},
{name: "GT", s: `>`, tok: pql.GT, lit: `>`},
{name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`},
{name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`},
{name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`},
{name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`},
{name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`},
{name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`},
{name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`},
{name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`},
{name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`},
{name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`},
{name: "ALL", s: `all`, tok: pql.ALL, lit: `all`},
{name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive
}
for i, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := pql.NewScanner(strings.NewReader(tt.s))
tok, pos, lit := s.Scan()
if tt.tok != tok {
t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit)
} else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char {
t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos)
} else if tt.lit != lit {
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit)
}
})
}
}

58
pql/token.go
View file

@ -14,28 +14,12 @@
package pql
import "strings"
// Token is a lexical token of the PQL language.
type Token int
const (
// Special tokens
ILLEGAL Token = iota
EOF
WS
literal_beg
IDENT // main
STRING // "foo"
BADSTRING // bad escape or unclosed string
INTEGER // 12345
FLOAT // 100.2
literal_end
keyword_beg
ALL
keyword_end
ASSIGN // =
EQ // ==
@ -45,23 +29,10 @@ const (
GT // >
GTE // >=
BETWEEN // ><
COMMA // ,
LPAREN // (
RPAREN // )
LBRACK // (
RBRACK // )
)
var tokens = [...]string{
ILLEGAL: "ILLEGAL",
EOF: "EOF",
WS: "WS",
IDENT: "IDENT",
INTEGER: "INTEGER",
FLOAT: "FLOAT",
ALL: "ALL",
ASSIGN: "=",
EQ: "==",
@ -71,20 +42,6 @@ var tokens = [...]string{
GT: ">",
GTE: ">=",
BETWEEN: "><",
COMMA: ",",
LPAREN: "(",
RPAREN: ")",
LBRACK: "(",
RBRACK: ")",
}
var keywords map[string]Token
func init() {
keywords = make(map[string]Token)
for tok := keyword_beg + 1; tok < keyword_end; tok++ {
keywords[strings.ToLower(tokens[tok])] = tok
}
}
// String returns the string representation of the token.
@ -94,18 +51,3 @@ func (tok Token) String() string {
}
return ""
}
// Lookup returns the token associated with a given string.
func Lookup(ident string) Token {
if tok, ok := keywords[strings.ToLower(ident)]; ok {
return tok
}
return IDENT
}
// Pos specifies the line and character position of a token.
// The Char and Line are both zero-based indexes.
type Pos struct {
Line int
Char int
}

7
roaring/containers.go
View file

@ -132,6 +132,13 @@ func (sc *SliceContainers) Count() uint64 {
return n
}
func (sc *SliceContainers) Reset() {
sc.keys = sc.keys[:0]
sc.containers = sc.containers[:0]
sc.lastContainer = nil
sc.lastKey = 0
}
func (sc *SliceContainers) seek(key uint64) (int, bool) {
i := search64(sc.keys, key)
found := true

5
roaring/roaring.go
View file

@ -94,6 +94,8 @@ type Containers interface {
// container is found at key.
Iterator(key uint64) (citer ContainerIterator, found bool)
Count() uint64
//Reset will clear the containers collection to allow for recycling during snapshot
Reset()
}
type ContainerIterator interface {
@ -631,7 +633,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
keyN := binary.LittleEndian.Uint32(data[4:8])
headerSize := headerBaseSize
b.Containers.Reset()
// Descriptive header section: Read container keys and cardinalities.
for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
b.Containers.PutContainerValues(
@ -688,6 +690,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
// FIXME(benbjohnson): return error with position so file can be trimmed.
return err
}
opr.apply(b)
// Increase the op count.

63
server.go
View file

@ -61,12 +61,10 @@ type Server struct {
clusterDisabled bool
// External
handler Handler
BroadcastReceiver BroadcastReceiver
systemInfo SystemInfo
gcNotifier GCNotifier
logger Logger
ln net.Listener
NodeID string
URI URI
@ -81,6 +79,11 @@ type Server struct {
dataDir string
}
// TODO: have this return an interface for Holder instead of concrete object?
func (s *Server) Holder() *Holder {
return s.holder
}
// ServerOption is a functional option type for pilosa.Server
type ServerOption func(s *Server) error
@ -126,13 +129,6 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption {
}
}
func OptServerHandler(h Handler) ServerOption {
return func(s *Server) error {
s.handler = h
return nil
}
}
func OptServerMaxWritesPerRequest(n int) ServerOption {
return func(s *Server) error {
s.maxWritesPerRequest = n
@ -191,14 +187,6 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
}
}
func OptServerListener(ln net.Listener) ServerOption {
return func(s *Server) error {
s.ln = ln
return nil
}
}
func OptServerURI(uri *URI) ServerOption {
return func(s *Server) error {
s.URI = *uri
@ -261,11 +249,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.translateFile.Path = filepath.Join(path, ".keys")
s.translateFile.PrimaryTranslateStore = s.primaryTranslateStore
// update URI port with actual listener port. TODO this should probably be done outside of here.
if s.URI.Port() == 0 {
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
}
// Get or create NodeID.
s.NodeID = s.LoadNodeID()
// Set Cluster Node.
@ -290,8 +273,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.executor.Cluster = s.Cluster
s.executor.TranslateStore = s.translateFile
s.executor.MaxWritesPerRequest = s.maxWritesPerRequest
s.handler.GetAPI().Executor = s.executor
s.handler.GetAPI().TranslateStore = s.translateFile
return s, nil
}
@ -299,9 +280,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
// Open opens and initializes the server.
func (s *Server) Open() error {
s.logger.Printf("open server")
if s.ln == nil {
return errors.New("must pass a listener option to NewServer")
}
// Log startup
err := s.holder.logStartup()
@ -318,20 +296,9 @@ func (s *Server) Open() error {
s.Cluster.Broadcaster = s
s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest
// Initialize HTTP handler.
api := s.handler.GetAPI()
api.Holder = s.holder
api.Broadcaster = s
api.BroadcastHandler = s
api.StatusHandler = s
api.Cluster = s.Cluster
// Initialize Holder.
s.holder.Broadcaster = s
// Serve handler.
go s.handler.Serve(s.ln, s.closing)
// Start the BroadcastReceiver.
if err := s.BroadcastReceiver.Start(s); err != nil {
return fmt.Errorf("starting BroadcastReceiver: %v", err)
@ -372,9 +339,6 @@ func (s *Server) Close() error {
close(s.closing)
s.wg.Wait()
if s.ln != nil {
s.ln.Close()
}
if s.Cluster != nil {
s.Cluster.close()
}
@ -402,12 +366,21 @@ func (s *Server) LoadNodeID() string {
return nodeID
}
type pilosaAddr URI
func (p pilosaAddr) String() string {
uri := URI(p)
return uri.HostPort()
}
func (pilosaAddr) Network() string {
return "tcp"
}
// Addr returns the address of the listener.
func (s *Server) Addr() net.Addr {
if s.ln == nil {
return nil
}
return s.ln.Addr()
return pilosaAddr(s.URI)
}
func (s *Server) monitorAntiEntropy() {

14
server/cluster_test.go
View file

@ -92,8 +92,8 @@ func TestMain_SendReceiveMessage(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
Set(1, f=1)
Set(2400000, f=1)
`); err != nil {
t.Fatal(err)
}
@ -259,8 +259,8 @@ func TestClusterResize_AddNode(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=1300000)
Set(1, f=1)
Set(1300000, f=1)
`); err != nil {
t.Fatal(err)
}
@ -311,8 +311,8 @@ func TestClusterResize_AddNode(t *testing.T) {
// Write data on first node. Note that no data is placed on slice 1.
if _, err := m0.Query("i", "", `
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
Set(1, f=1)
Set(2400000, f=1)
`); err != nil {
t.Fatal(err)
}
@ -466,7 +466,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// TODO: Deterministic node IDs would ensure consistent results
setColumns := ""
for i := 0; i < 20; i++ {
setColumns += fmt.Sprintf("SetBit(row=1, field=\"f\", col=%d) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("Set(%d, f=1) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setColumns); err != nil {

2
server/config.go
View file

@ -47,7 +47,7 @@ type Config struct {
Bind string `toml:"bind"`
// MaxWritesPerRequest limits the number of mutating commands that can be in
// a single request to the server. This includes SetBit, ClearBit,
// a single request to the server. This includes Set, Clear,
// SetRowAttrs & SetColumnAttrs.
MaxWritesPerRequest int `toml:"max-writes-per-request"`

599
server/handler_test.go Normal file
View file

@ -0,0 +1,599 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server_test
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http/httptest"
"reflect"
"strings"
"testing"
gohttp "net/http"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/test"
)
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_Endpoints(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
t.Run("Not Found", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != gohttp.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
})
t.Run("Info", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) {
t.Fatalf("unexpected body: %s", body)
}
})
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
t.Run("Schema", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("Status", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
ret := mustJSONDecode(t, w.Body)
if ret["state"].(string) != "NORMAL" {
t.Fatalf("wrong state from /status: %#v", ret)
}
if len(ret["nodes"].([]interface{})) != 1 {
t.Fatalf("wrong length nodes list: %#v", ret)
}
})
t.Run("Abort no resize job", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != gohttp.StatusInternalServerError {
bod, err := ioutil.ReadAll(w.Body)
t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err)
}
// TODO need to test aborting a cluster resize job. this may not be the right place
})
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4)
hldr.SetBit("i0", "f0", 31, 1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8)
t.Run("Max Slice", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("Slices args", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[2]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Slices args protobuf", func(t *testing.T) {
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
Query: "Count(Row(f0=30))",
Slices: []uint64{0, 1},
})
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
req := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[2]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query args error", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query params err", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Uint64 protobuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Count(Row(f0=30))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 3 {
t.Fatalf("unexpected n: %d", n)
}
})
t.Run("Row JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
f0 := i0.Field("f0")
if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
} else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil {
t.Fatal(err)
}
t.Run("ColumnAttrs_JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("Row pbuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
})
t.Run("Row columnattrs protobuf", func(t *testing.T) {
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
Query: "Row(f0=30)",
ColumnAttrs: true,
})
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(buf))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
if a := resp.ColumnAttrSets; len(a) != 2 {
t.Fatalf("unexpected column attributes length: %d", len(a))
} else if a[0].ID != pilosa.SliceWidth+1 {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
}
})
t.Run("Query Pairs JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`)))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query Pairs protobuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
})
t.Run("Query err JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`)))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"executing: field not found"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query err protobuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.Err; s != `executing: field not found` {
t.Fatalf("unexpected error: %s", s)
}
})
t.Run("Method not allowed", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil))
if w.Code != gohttp.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
})
t.Run("Err Parse", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("delete index", func(t *testing.T) {
hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if w.Body.String() != "{}\n" {
t.Fatalf("unexpected response body: %s", w.Body.String())
}
// Verify index is gone.
if hldr.Index("i") != nil {
t.Fatal("expected nil index")
}
})
t.Run("Field delete", func(t *testing.T) {
i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := i.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
})
i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err := i.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := i.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := i.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
t.Run("AttrStore Diff", func(t *testing.T) {
blks, err := i.ColumnAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req := test.MustNewHTTPRequest(
"POST",
"/index/i/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
}
// Read and validate body.
if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", w.Body.String())
}
})
meta, err := i.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
}
if err := meta.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := meta.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := meta.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
t.Run("field attrstore diff", func(t *testing.T) {
blks, err := meta.RowAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req := test.MustNewHTTPRequest(
"POST",
"/index/i/field/meta/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
}
// Read and validate body.
if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", w.Body.String())
}
})
t.Run("Version", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
version := strings.TrimPrefix(pilosa.Version, "v")
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"`+version+`"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
})
t.Run("Fragment Nodes", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
body := mustJSONDecodeSlice(t, w.Body)
bmap := body[0].(map[string]interface{})
if bmap["isCoordinator"] != true {
t.Fatalf("expected true coordinator")
}
// invalid argument should return BadRequest
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
// index is required
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("Expvars", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("Recalculate Caches", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
if w.Code != gohttp.StatusNoContent {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("CORS", func(t *testing.T) {
req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil)
req.Header.Add("Origin", "http://test/")
req.Header.Add("Access-Control-Request-Method", "POST")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
result := w.Result()
// This handler does not support CORS, return Method Not Allowed (405)
if result.StatusCode != 405 {
t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode)
}
clus := test.MustRunMainWithCluster(t, 1, test.OptAllowedOrigins([]string{"http://test/"}))
w = httptest.NewRecorder()
h := clus[0].Handler.(*http.Handler).Handler
h.ServeHTTP(w, req)
result = w.Result()
if result.StatusCode != 200 {
t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode)
}
if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" {
t.Fatal("CORS header not present")
}
})
}
func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) {
dec := json.NewDecoder(r)
err := dec.Decode(&ret)
if err != nil {
t.Fatalf("decoding response: %v", err)
}
return ret
}
func mustJSONDecodeSlice(t *testing.T, r io.Reader) (ret []interface{}) {
dec := json.NewDecoder(r)
err := dec.Decode(&ret)
if err != nil {
t.Fatalf("decoding response: %v", err)
}
return ret
}

57
server/server.go
View file

@ -73,6 +73,10 @@ type Command struct {
// Passed to the Gossip implementation.
logOutput io.Writer
logger loggerLogger
Handler pilosa.Handler
API *pilosa.API
ln net.Listener
}
// NewCommand returns a new instance of Main.
@ -102,6 +106,12 @@ func (m *Command) Start() (err error) {
if err != nil {
return errors.Wrap(err, "setting up networking")
}
go func() {
err := m.Handler.Serve()
if err != nil {
m.logger.Printf("Handler serve error: %v", err)
}
}()
// Initialize server.
if err = m.Server.Open(); err != nil {
@ -164,18 +174,6 @@ func (m *Command) SetupServer() error {
}
m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime)
api := pilosa.NewAPI()
api.Logger = m.logger
handler, err := http.NewHandler(
http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
http.OptHandlerAPI(api),
http.OptHandlerLogger(m.logger),
)
if err != nil {
return errors.Wrap(err, "wrapping handler")
}
uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
if err != nil {
return errors.Wrap(err, "processing bind address")
@ -210,11 +208,16 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "new stats client")
}
ln, err := getListener(*uri, TLSConfig)
m.ln, err = getListener(*uri, TLSConfig)
if err != nil {
return errors.Wrap(err, "getting listener")
}
// If port is 0, get auto-allocated port from listener
if uri.Port() == 0 {
uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
}
c := http.GetHTTPClient(TLSConfig)
// Setup connection to primary store if this is a replica.
@ -234,18 +237,31 @@ func (m *Command) SetupServer() error {
pilosa.OptServerLogger(m.logger),
pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
pilosa.OptServerHandler(handler),
pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
pilosa.OptServerStatsClient(statsClient),
pilosa.OptServerListener(ln),
pilosa.OptServerURI(uri),
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore),
pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
)
if err != nil {
return errors.Wrap(err, "new server")
}
m.API, err = pilosa.NewAPI(pilosa.OptAPIServer(m.Server))
if err != nil {
return errors.Wrap(err, "new api")
}
m.Handler, err = http.NewHandler(
http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
http.OptHandlerAPI(m.API),
http.OptHandlerLogger(m.logger),
http.OptHandlerListener(m.ln),
)
return errors.Wrap(err, "new handler")
return errors.Wrap(err, "new server")
}
// SetupNetworking sets up internode communication based on the configuration.
@ -300,17 +316,16 @@ func (m *Command) SetupNetworking() error {
// Close shuts down the server.
func (m *Command) Close() error {
var logErr error
handlerErr := m.Handler.Close()
serveErr := m.Server.Close()
if closer, ok := m.logOutput.(io.Closer); ok {
logErr = closer.Close()
}
close(m.done)
if serveErr != nil && logErr != nil {
return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr)
} else if logErr != nil {
return logErr
if serveErr != nil || logErr != nil || handlerErr != nil {
return fmt.Errorf("closing server: '%v', closing logs: '%v', closing handler: '%v'", serveErr, logErr, handlerErr)
}
return serveErr
return nil
}
// NewStatsClient creates a stats client from the config

48
server/server_test.go
View file

@ -49,7 +49,7 @@ func TestMain_Set_Quick(t *testing.T) {
t.Fatal(err)
}
// Execute SetBit() commands.
// Execute Set() commands.
for _, cmd := range cmds {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
@ -57,7 +57,7 @@ func TestMain_Set_Quick(t *testing.T) {
if err := client.CreateField(context.Background(), "i", cmd.Field, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists {
t.Fatal(err)
}
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, field=%q, col=%d)`, cmd.ID, cmd.Field, cmd.ColumnID)); err != nil {
if _, err := m.Query("i", "", fmt.Sprintf(`Set(%d, %s=%d)`, cmd.ColumnID, cmd.Field, cmd.ID)); err != nil {
t.Fatal(err)
}
}
@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Row(%s=%d)`, field, id)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -132,36 +132,36 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Set columns on different rows in different fields.
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, field="x", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, field="z", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `Set(100, x=2)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=3, field="neg", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `Set(100, neg=3)`); err != nil {
t.Fatal(err)
}
// Set row attributes.
if _, err := m.Query("i", "", `SetRowAttrs(row=1, field="x", x=100)`); err != nil {
if _, err := m.Query("i", "", `SetRowAttrs(x, 1, x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="x", x=-200)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(x, 2, x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="z", x=300)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(z, 2, x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=3, field="neg", x=-0.44)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(neg, 3, x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
if res, err := m.Query("i", "", `Row(x=2)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, field="neg")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(neg=3)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
if res, err := m.Query("i", "", `Row(x=2)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -205,19 +205,19 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Set columns on row.
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
if _, err := m.Query("i", "", `Set(100, x=1)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=101)`); err != nil {
} else if _, err := m.Query("i", "", `Set(101, x=1)`); err != nil {
t.Fatal(err)
}
// Set column attributes.
if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
if _, err := m.Query("i", "", `SetColumnAttrs(100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Row(x=1)`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
@ -279,7 +279,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
data = append(data, fmt.Sprintf(`SetBit(row=%d, field="f", col=%d)`, rowID, columnID))
data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID))
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
@ -296,7 +296,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster {
res, err := m.Query("i", "", `TopN(field="f")`)
res, err := m.Query("i", "", `TopN(f)`)
if err != nil {
t.Fatal(err)
}

4
server_test.go
View file

@ -27,7 +27,7 @@ import (
// pilosa.Server was not having its remoteClient field set by an option and so
// it was using a nil client in monitorAntiEntropy.
func TestMonitorAntiEntropy(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1))
cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*20))
client := cluster[1].Client()
err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{})
if err != nil {
@ -38,7 +38,7 @@ func TestMonitorAntiEntropy(t *testing.T) {
t.Fatalf("creating field: %v", err)
}
time.Sleep(time.Millisecond * 2)
time.Sleep(time.Millisecond * 40)
for _, m := range cluster {
err := m.Close()
if err != nil {

190
stats_test.go
View file

@ -16,12 +16,13 @@ package pilosa_test
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/test"
)
@ -127,8 +128,8 @@ func TestStatsCount_Bitmap(t *testing.T) {
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "Bitmap" {
t.Errorf("Expected Bitmap, Results %s", name)
if name != "Row" {
t.Errorf("Expected Row, Results %s", name)
}
if tags[0] != "index:d" {
@ -138,7 +139,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(field=f, row=0)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Row(f=0)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -168,7 +169,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -199,7 +200,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, field=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(10, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -207,116 +208,89 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
}
}
func TestStatsCount_CreateIndex(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "createIndex" {
t.Errorf("Expected createIndex, Results %s", name)
}
func TestStatsCount_APICalls(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", nil))
if !called {
t.Error("Count isn't called")
}
}
t.Run("create index", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "createIndex" {
t.Errorf("Expected createIndex, Results %s", name)
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
func TestStatsCount_DeleteIndex(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
t.Run("create field", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "createField" {
t.Errorf("Expected createField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
// Create index.
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "deleteIndex" {
t.Errorf("Expected deleteIndex, Results %s", name)
}
t.Run("delete field", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "deleteField" {
t.Errorf("Expected deleteField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
func TestStatsCount_CreateField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
t.Run("delete index", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "deleteIndex" {
t.Errorf("Expected deleteIndex, Results %s", name)
}
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
// Create index.
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "createField" {
t.Errorf("Expected createField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", nil))
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called := false
// Create index.
indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := indx.CreateFieldIfNotExists("test", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "deleteField" {
t.Errorf("Expected deleteField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
}
type MockStats struct {

32
test/handler.go
View file

@ -45,9 +45,6 @@ func NewHandler(opts ...http.HandlerOption) (*Handler, error) {
h := &Handler{
Handler: handler,
}
h.API = pilosa.NewAPI()
h.Handler.API = h.API
h.Handler.API.Executor = &h.Executor
// Handler test messages can no-op.
h.API.Broadcaster = pilosa.NopBroadcaster
@ -84,22 +81,23 @@ type Server struct {
// NewServer returns a test server running on a random port.
func NewServer() *Server {
handler, err := NewHandler()
if err != nil {
panic(err)
}
s := &Server{
Handler: handler,
}
s.Server = httptest.NewServer(s.Handler.Handler)
return &Server{}
//handler, err := pilosa.NewHandler()
//if err != nil {
// panic(err)
//}
//s := &Server{
// Handler: handler,
//}
//s.Server = httptest.NewServer(s.Handler.Handler)
// Handler test messages can no-op.
s.Handler.API.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.API.Cluster = NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
//// Handler test messages can no-op.
//s.Handler.API.Broadcaster = pilosa.NopBroadcaster
//// Create a default cluster on the handler
//s.Handler.API.Cluster = NewCluster(1)
//s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
return s
//return s
}
// LocalStatus exists so that test.Server implements StatusHandler.

15
test/pilosa.go
View file

@ -19,6 +19,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
gohttp "net/http"
"os"
"strings"
@ -51,6 +52,13 @@ func OptAntiEntropyInterval(dur time.Duration) MainOpt {
}
}
func OptAllowedOrigins(origins []string) MainOpt {
return func(m *Main) error {
m.Config.Handler.AllowedOrigins = origins
return nil
}
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain(opts ...MainOpt) *Main {
path, err := ioutil.TempDir("", "pilosa-")
@ -221,6 +229,13 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (
m.Server.Cluster.Static = false
go func() {
err := m.Handler.Serve()
if err != nil {
log.Printf("Handler serve error: %v", err)
}
}()
// Initialize server.
err = m.Server.Open()
if err != nil {