diff --git a/Makefile b/Makefile index 363bcd6b6..90e4700df 100644 --- a/Makefile +++ b/Makefile @@ -48,8 +48,8 @@ build: vendor # Create a single release build under the build directory release-build: vendor $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1 - cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID) - $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID)) + cp NOTICE README.md build/pilosa-$(VERSION_ID) + $(if $(ENTERPRISE_ENABLED),cp enterprise/COPYING build/pilosa-$(VERSION_ID),cp LICENSE build/pilosa-$(VERSION_ID)) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz diff --git a/executor_test.go b/executor_test.go index 7a0991288..4aa3ed192 100644 --- a/executor_test.go +++ b/executor_test.go @@ -19,13 +19,14 @@ import ( "fmt" "reflect" "strconv" + "strings" "testing" "github.com/davecgh/go-spew/spew" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" "github.com/pkg/errors" ) @@ -33,71 +34,70 @@ import ( // Ensure a bitmap query can be executed. func TestExecutor_Execute_Bitmap(t *testing.T) { t.Run("Row", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) f, err := index.CreateField("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - ), nil, nil); err != nil { + }); err != nil { t.Fatal(err) } if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit column attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeColumns: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit row attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeRowAttrs: true}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("Column", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10)+ - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10)+ + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - ), nil, nil); err != nil { + }); err != nil { t.Fatal(err) } if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { @@ -106,28 +106,33 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { }) t.Run("Keys", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - - // Set bits. - if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - `Set("foo", f="bar")`+"\n"+ - `Set("foo", f="baz")`+"\n"+ - `Set("bat", f="bar")`+"\n"+ - `Set("aaa", f="bbb")`+"\n", - ), nil, nil); err != nil { - t.Fatal(err) + _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `` + + `Set("foo", f="bar")` + "\n" + + `Set("foo", f="baz")` + "\n" + + `Set("bat", f="bar")` + "\n" + + `Set("aaa", f="bbb")` + "\n", + }) + if err != nil { + t.Fatalf("querying: %v", err) } - if results, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f="bar")`), nil, nil); err != nil { + if results, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `Row(f="bar")`, + }); err != nil { t.Fatal(err) - } else if diff := cmp.Diff(results, []interface{}{ + } else if diff := cmp.Diff(results.Results, []interface{}{ &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { t.Fatal(diff) @@ -137,38 +142,39 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Ensure a difference query can be executed. func TestExecutor_Execute_Difference(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 1) hldr.SetBit("i", "general", 10, 2) hldr.SetBit("i", "general", 10, 3) hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, 4) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty difference query behaves properly. func TestExecutor_Execute_Empty_Difference(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 1) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference()`}); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } } // Ensure an intersect query can be executed. func TestExecutor_Execute_Intersect(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 1) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -177,29 +183,28 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty intersect query behaves properly. func TestExecutor_Execute_Empty_Intersect(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect()`}); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } } // Ensure a union query can be executed. func TestExecutor_Execute_Union(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 0) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -207,32 +212,33 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure an empty union query behaves properly. func TestExecutor_Execute_Empty_Union(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 0) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union()`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure a xor query can be executed. func TestExecutor_Execute_Xor(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -240,27 +246,27 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Row(general=10), Row(general=11))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } } // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "f", 10, 3) hldr.SetBit("i", "f", 10, ShardWidth+1) hldr.SetBit("i", "f", 10, ShardWidth+2) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { t.Fatal(err) - } else if res[0] != uint64(3) { - t.Fatalf("unexpected n: %d", res[0]) + } else if res.Results[0] != uint64(3) { + t.Fatalf("unexpected n: %d", res.Results[0]) } } @@ -370,24 +376,24 @@ func TestExecutor_Execute_SetBit(t *testing.T) { // Ensure old PQL syntax doesn't break anything too badly. func TestExecutor_Execute_OldPQL(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // set a bit so the view gets created. hldr.SetBit("i", "f", 1, 0) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(frame=f, row=11, col=1)`), nil, nil); err == nil || err.Error() != "unknown call: SetBit" { - t.Fatal("Expected error: 'unknown call: SetBit'") + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { + t.Fatalf("Expected error: 'unknown call: SetBit', got: %v", errors.Cause(err)) } } // Ensure a SetValue() query can be executed. func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Create felds. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -402,10 +408,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } // Set bsiGroup values. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f=25)`}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil { + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=100, f=10)`}); err != nil { t.Fatal(err) } @@ -428,8 +433,10 @@ func TestExecutor_Execute_SetValue(t *testing.T) { }) t.Run("", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, @@ -440,22 +447,19 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(invalid_column_name="bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetValue(col=10, f="hello")`}); err == nil || errors.Cause(err) != pilosa.ErrInvalidBSIGroupValueType { t.Fatalf("unexpected error: %s", err) } }) @@ -464,8 +468,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Ensure a SetRowAttrs() query can be executed. func TestExecutor_Execute_SetRowAttrs(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -477,17 +482,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two attrs on f/10. // Also set attrs on other bitmaps and fields to test isolation. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, foo="bar")`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 200, YYY=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(xxx, 10, YYY=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil { t.Fatal(err) } @@ -502,9 +506,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { t.Run("ID", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { @@ -513,17 +517,17 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) Set(1, f=0) - Set(`+strconv.Itoa(ShardWidth)+`, f=0) - Set(`+strconv.Itoa(ShardWidth+2)+`, f=0) - Set(`+strconv.Itoa((5*ShardWidth)+100)+`, f=0) + Set(` + strconv.Itoa(ShardWidth) + `, f=0) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=0) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=0) Set(0, f=10) - Set(`+strconv.Itoa(ShardWidth)+`, f=10) - Set(`+strconv.Itoa(ShardWidth)+`, f=20) + Set(` + strconv.Itoa(ShardWidth) + `, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=20) Set(0, other=0) - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } @@ -531,9 +535,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ {ID: 0, Count: 5}, {ID: 10, Count: 2}, }) { @@ -542,9 +546,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { }) t.Run("Keys", func(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { @@ -553,7 +557,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("a", f="foo") Set("b", f="foo") Set("c", f="foo") @@ -563,15 +567,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { Set("b", f="bar") Set("b", f="baz") Set("a", other="foo") - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=2)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if diff := cmp.Diff(result, []interface{}{ + } else if diff := cmp.Diff(result.Results, []interface{}{ []pilosa.Pair{ {Key: "foo", Count: 5}, {Key: "bar", Count: 2}, @@ -583,8 +587,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { } func TestExecutor_Execute_TopN_fill(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) @@ -595,10 +600,9 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr.SetBit("i", "f", 1, ShardWidth) // Execute query. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -607,8 +611,9 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Ensure func TestExecutor_Execute_TopN_fill_small(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, ShardWidth) @@ -629,10 +634,9 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr.SetBit("i", "f", 4, 3*ShardWidth+1) // Execute query. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -641,8 +645,9 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Ensure a TopN() query with a source bitmap can be executed. func TestExecutor_Execute_TopN_Src(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) @@ -664,10 +669,9 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() // Execute query. - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(other=100), n=3)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, {ID: 10, Count: 2}, {ID: 0, Count: 1}, @@ -678,9 +682,9 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { //Ensure TopN handles Attribute filters func TestExecutor_Execute_TopN_Attr(t *testing.T) { - // - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", 10, ShardWidth) @@ -688,10 +692,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -701,9 +704,10 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { //Ensure TopN handles Attribute filters with source bitmap func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { - // - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", 10, ShardWidth) @@ -711,10 +715,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ + } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -723,9 +726,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // Ensure Min() and Max() queries can be executed. func TestExecutor_Execute_MinMax(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -744,22 +747,22 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, x=0) Set(3, x=0) - Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) Set(1, x=1) - Set(`+strconv.Itoa(ShardWidth+2)+`, x=2) + Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) SetValue(col=0, f=20) SetValue(col=1, f=-5) SetValue(col=2, f=-5) SetValue(col=3, f=10) - SetValue(col=`+strconv.Itoa(ShardWidth)+`, f=30) - SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, f=40) - SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, f=50) - SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, f=60) - `), nil, nil); err != nil { + SetValue(col=` + strconv.Itoa(ShardWidth) + `, f=30) + SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, f=40) + SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, f=60) + `}); err != nil { t.Fatal(err) } @@ -781,9 +784,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } else { pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } - if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -807,9 +810,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } else { pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } - if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -818,9 +821,9 @@ func TestExecutor_Execute_MinMax(t *testing.T) { // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -855,33 +858,33 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, x=0) - Set(`+strconv.Itoa(ShardWidth+1)+`, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) SetValue(col=0, foo=20) SetValue(col=0, bar=2000) - SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=40) - SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=50) - SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) + SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) + SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=40) + SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) SetValue(col=0, other=1000) - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(field=foo)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Row(x=0), field=foo)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -889,9 +892,9 @@ func TestExecutor_Execute_Sum(t *testing.T) { // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -905,7 +908,7 @@ func TestExecutor_Execute_Range(t *testing.T) { } // Set columns. - cc := test.MustParse(` + cc := ` Set(2, f=1, 1999-12-31T00:00) Set(3, f=1, 2000-01-01T00:00) Set(4, f=1, 2000-01-02T00:00) @@ -916,27 +919,27 @@ func TestExecutor_Execute_Range(t *testing.T) { Set(2, f=1, 1999-12-30T00:00) Set(2, f=1, 2002-02-01T00:00) Set(2, f=10, 2001-01-01T00:00) - `) - if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { + ` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) t.Run("Clear", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Clear( 2, f=1)`), nil, nil); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f=1)`}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) @@ -944,9 +947,9 @@ func TestExecutor_Execute_Range(t *testing.T) { // Ensure a Range(bsiGroup) query can be executed. func TestExecutor_Execute_BSIGroupRange(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -989,136 +992,136 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) - Set(`+strconv.Itoa(ShardWidth+1)+`, f=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) SetValue(col=50, foo=20) SetValue(col=50, bar=2000) - SetValue(col=`+strconv.Itoa(ShardWidth)+`, foo=30) - SetValue(col=`+strconv.Itoa(ShardWidth+2)+`, foo=10) - SetValue(col=`+strconv.Itoa((5*ShardWidth)+100)+`, foo=20) - SetValue(col=`+strconv.Itoa(ShardWidth+1)+`, foo=60) + SetValue(col=` + strconv.Itoa(ShardWidth) + `, foo=30) + SetValue(col=` + strconv.Itoa(ShardWidth+2) + `, foo=10) + SetValue(col=` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + SetValue(col=` + strconv.Itoa(ShardWidth+1) + `, foo=60) SetValue(col=0, other=1000) SetValue(col=0, edge=100) SetValue(col=1, edge=-100) - `), nil, nil); err != nil { + `}); err != nil { t.Fatal(err) } t.Run("EQ", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != null)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != null)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo != 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != -20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != -20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns()) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) } }) t.Run("LT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo < 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("LTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo <= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("GT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo > 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("GTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo >= 20)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(0 < other < 1000)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(-1 < other < 1000)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(-1 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("BelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 0)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 0)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("AboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge < 200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge < 200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) } }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge > -200)`), nil, nil); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -200)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) } }) t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatal(err) } }) @@ -1126,351 +1129,153 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) { // Ensure a remote query can return a row. func TestExecutor_Execute_Remote_Row(t *testing.T) { - t.Skip() // Until test.NewServer() works + c := test.MustRunCluster(t, 2, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + ) + defer c.Close() + hldr0 := test.Holder{Holder: c[0].Server.Holder()} + hldr1 := test.Holder{Holder: c[1].Server.Holder()} - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { - t.Fatal(err) + t.Fatalf("creating index: %v", err) } - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Row(f=10)` { - t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(shards, []uint64{1}) { - t.Fatalf("unexpected shards: %+v", shards) - } - - // Set columns in shard 0 & 2. - r := pilosa.NewRow( - (0*ShardWidth)+1, - (0*ShardWidth)+2, - (2*ShardWidth)+4, - ) - return []interface{}{r}, nil + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field: %v", err) } - // Create local executor data. - // The local node owns shard 1. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr1.MustSetBits("i", "f", 10, ShardWidth+1, ShardWidth+2, (3*ShardWidth)+4) + hldr0.SetBit("i", "f", 10, 1) - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Row(f=10)`), nil, nil); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*ShardWidth + 4}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 1, ShardWidth + 2, (3 * ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } -} -// Ensure a remote query can return a count. -func TestExecutor_Execute_Remote_Count(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return []interface{}{uint64(10)}, nil - } - - // Create local executor data. The local node owns shard 1. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, (2*ShardWidth)+1) - hldr.SetBit("i", "f", 10, (2*ShardWidth)+2) - - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Row(f=10))`), nil, nil); err != nil { - t.Fatal(err) - } else if res[0] != uint64(12) { - t.Fatalf("unexpected n: %d", res[0]) - } -} - -// Ensure a remote query can set columns on multiple nodes. -func TestExecutor_Execute_Remote_SetBit(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - c.ReplicaN = 2 - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments. - var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != `i` { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Set(_col=2, f=10)` { - t.Fatalf("unexpected query: %s", query.String()) + t.Run("Count", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { + t.Fatal(err) + } else if res.Results[0] != uint64(4) { + t.Fatalf("unexpected n: %d", res.Results[0]) } - remoteCalled = true - return []interface{}{nil}, nil - } + }) - // Create local executor data. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - - // Create field. - if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - - e := test.NewExecutor(hldr.Holder, c) - cc := test.MustParse("Set(2, f=10)") - if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { - t.Fatal(err) - } - - // Verify that one column is set on both node's holder. - if n := hldr.Row("i", "f", 10).Count(); n != 1 { - t.Fatalf("unexpected local count: %d", n) - } - if !remoteCalled { - t.Fatalf("expected remote execution") - } -} - -// Ensure a remote query can set columns on multiple nodes. -func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - c.ReplicaN = 2 - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments. - var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != `i` { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Set(_col=2, _timestamp="2016-12-11T10:09", f=10)` { - t.Fatalf("unexpected query: %s", query.String()) - } - remoteCalled = true - return []interface{}{nil}, nil - } - - // Create local executor data. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - - // Create field. - if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } else if err := f.SetTimeQuantum("Y"); err != nil { - t.Fatal(err) - } - - e := test.NewExecutor(hldr.Holder, c) - cc := test.MustParse(`Set(2, f=10, 2016-12-11T10:09)`) - if _, err := e.Execute(context.Background(), "i", cc, nil, nil); err != nil { - t.Fatal(err) - } - - // Verify that one column is set on both node's holder. - if n := hldr.ViewRow("i", "f", "standard_2016", 10).Count(); n != 1 { - t.Fatalf("unexpected local count: %d", n) - } - if !remoteCalled { - t.Fatalf("expected remote execution") - } -} - -// Ensure a remote query can return a top-n query. -func TestExecutor_Execute_Remote_TopN(t *testing.T) { - t.Skip() // Until test.NewServer() works - - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - var remoteExecN int - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if !reflect.DeepEqual(shards, []uint64{1, 3}) { - t.Fatalf("unexpected shards: %+v", shards) + t.Run("Remote SetBit", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil { + t.Fatalf("quuerying remote: %v", err) } - // Query should be executed twice. Once to get the top bitmaps for the - // shards and a second time to get the counts for a set of bitmaps. - switch remoteExecN { - case 0: - if query.String() != `TopN(_field="f", n=3)` { - t.Fatalf("unexpected query(0): %s", query.String()) - } - case 1: - if query.String() != `TopN(_field="f", ids=[0,10,30], n=3)` { - t.Fatalf("unexpected query(1): %s", query.String()) - } - default: - t.Fatalf("too many remote exec calls") + if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) { + t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns()) } - remoteExecN++ + }) - // Return pair counts. - return []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, - {ID: 30, Count: 2}, - }}, nil - } - - // Create local executor data on shard 2 & 4. - hldr := test.MustOpenHolder() - defer hldr.Close() - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 30, (2*ShardWidth)+1) - hldr.SetBit("i", "f", 30, (4*ShardWidth)+2) - - e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(f, n=3)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 30, Count: 4}, - {ID: 10, Count: 2}, - }}) { - t.Fatalf("unexpected results: %s", spew.Sdump(res)) - } -} - -// Ensure a remote query can set RowAttrs -func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) { - t.Skip("test.NewServer broken") - c := pilosa.NewTestCluster(2) - - // Create secondary server and update second cluster node. - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.Host()) - if err != nil { - t.Fatal(err) - } - c.Nodes[1].URI = *uri - - // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if index != "i" { - t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)` { - t.Fatalf("unexpected query: %s", query.String()) + t.Run("remote with timestamp", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y")) + if err != nil { + t.Fatalf("creating field: %v", err) } - return []interface{}{}, nil - } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil { + t.Fatalf("quuerying remote: %v", err) + } - // Create local executor data. - // The local node owns shard 1. - hldr := test.MustOpenHolder() - defer hldr.Close() + if !reflect.DeepEqual(hldr1.ViewRow("i", "z", "standard_2010", 5).Columns(), []uint64{1500000}) { + t.Fatalf("unexpected cols from row 7: %v", hldr1.ViewRow("i", "z", "standard_2010", 5).Columns()) + } + }) - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { - t.Fatal(err) - } - f := hldr.Field("i", "f") - s.Handler.API.Holder = hldr.Holder - hldr.SetBit("i", "f", 10, ShardWidth+1) + t.Run("remote topn", func(t *testing.T) { + _, err = c[0].API.CreateField(context.Background(), "i", "fn", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } - e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(f, 10, baz=123, bat=true)`), nil, nil); err != nil { - t.Fatal(err) - } else if m, err := f.RowAttrStore().Attrs(10); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"bat": true, "baz": int64(123)}) { - t.Fatalf("unexpected bitmap attr: %#v", m) + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` +Set(500001, fn=5) +Set(1500001, fn=5) +Set(2500001, fn=5) +Set(3500001, fn=5) +Set(1500001, fn=3) +Set(1500002, fn=3) +Set(3500003, fn=3) +Set(500001, fn=4) +Set(4500001, fn=4) +`}); err != nil { + t.Fatalf("quuerying remote: %v", err) + } + err := c[0].API.RecalculateCaches(context.Background()) + if err != nil { + t.Fatalf("recalcing caches: %v", err) + } - } + if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `TopN(fn, n=3)`, + }); err != nil { + t.Fatalf("topn querying: %v", err) + } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ + {ID: 5, Count: 4}, + {ID: 3, Count: 3}, + {ID: 4, Count: 2}, + }}) { + t.Fatalf("topn wrong results: %v", res.Results) + } + }) + + t.Run("remote setrowattrs", func(t *testing.T) { + if _, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)`, + }); err != nil { + t.Fatalf("setrowattrs querying: %v", err) + } else if attrst, err := hldr0.RowAttrStore("i", "f").Attrs(10); err != nil || !attrst["bat"].(bool) || attrst["baz"].(int64) != 123 { + t.Fatalf("wrong attrs: %v", attrst) + } + }) } // Ensure executor returns an error if too many writes are in a single request. func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustNewCluster(t, 1) + c[0].Config.MaxWritesPerRequest = 3 + err := c.Start() + if err != nil { + t.Fatal(err) + } + hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - e.MaxWritesPerRequest = 3 - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Set() Clear() Set() Set()`), nil, nil); err != pilosa.ErrTooManyWrites { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set() Clear() Set() Set()`}); errors.Cause(err) != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) } } // Ensure SetColumnAttrs doesn't save `field` as an attribute func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateField("f", pilosa.FieldOptions{}) + _, err := index.CreateField("f", pilosa.FieldOptions{}) + if err != nil { + t.Fatalf("creating field: %v", err) + } targetAttrs := map[string]interface{}{ "foo": "bar", } - e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // SetColumnAttrs call should exclude the field attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("Set(10, f=1)"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(10, f=1)"}) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(10, foo='bar')"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(10, foo='bar')"}) if err != nil { t.Fatal(err) } @@ -1483,11 +1288,11 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { } // SetColumnAttrs call should not break if field is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("Set(20, f=10)"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(20, f=10)"}) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(20, foo='bar')"), nil, nil) + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(20, foo='bar')"}) if err != nil { t.Fatal(err) } @@ -1500,3 +1305,67 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { } } + +func TestExecutor_Time_Clear_Quantums(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + + var rangeTests = []struct { + quantum pilosa.TimeQuantum + expected []uint64 + }{ + {quantum: "Y", expected: []uint64{3, 4, 5, 6}}, + {quantum: "M", expected: []uint64{3, 4, 6}}, + {quantum: "D", expected: []uint64{3, 4, 5, 6}}, + {quantum: "H", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "YM", expected: []uint64{3, 4, 5, 6}}, + {quantum: "YMD", expected: []uint64{3, 4, 5, 6}}, + {quantum: "YMDH", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "MD", expected: []uint64{3, 4, 5, 6}}, + {quantum: "MDH", expected: []uint64{3, 4, 5, 6, 7}}, + {quantum: "DH", expected: []uint64{3, 4, 5, 6, 7}}, + } + populateBatch := test.MustParse(` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00) + `) + clearColumn := test.MustParse(`Clear( 2, f=1)`) + rangeCheckQuery := test.MustParse(`Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`) + + for i, tt := range rangeTests { + t.Run(fmt.Sprintf("#%d Quantum %s", i+1, tt.quantum), func(t *testing.T) { + // Create index. + indexName := strings.ToLower(string(tt.quantum)) + index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{}) + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeTime, + TimeQuantum: tt.quantum, + }); err != nil { + t.Fatal(err) + } + // Populate + if _, err := e.Execute(context.Background(), indexName, populateBatch, nil, nil); err != nil { + t.Fatal(err) + } + if _, err := e.Execute(context.Background(), indexName, clearColumn, nil, nil); err != nil { + t.Fatal(err) + } + if res, err := e.Execute(context.Background(), indexName, rangeCheckQuery, nil, nil); err != nil { + t.Fatal(err) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, tt.expected) { + t.Fatalf("unexpected columns: %+v", columns) + } + + }) + } + +} diff --git a/field.go b/field.go index 37cf17c2d..42f607ff9 100644 --- a/field.go +++ b/field.go @@ -583,7 +583,6 @@ func (f *Field) RecalculateCaches() { // CreateViewIfNotExists returns the named view, creating it if necessary. // Additionally, a CreateViewMessage is sent to the cluster. func (f *Field) CreateViewIfNotExists(name string) (*View, error) { - view, created, err := f.createViewIfNotExistsBase(name) if err != nil { return nil, err diff --git a/fragment.go b/fragment.go index 891c1570e..a5a40d68f 100644 --- a/fragment.go +++ b/fragment.go @@ -1752,7 +1752,7 @@ func (s *FragmentSyncer) syncFragment() error { } // Retrieve remote blocks. - blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.shard) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), &node.URI, s.Fragment.index, s.Fragment.field, s.Fragment.shard) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } diff --git a/holder.go b/holder.go index bfb2ae758..eb6eacd57 100644 --- a/holder.go +++ b/holder.go @@ -564,6 +564,8 @@ func (h *Holder) logStartup() error { // HolderSyncer is an active anti-entropy tool that compares the local holder // with a remote holder based on block checksums and resolves differences. type HolderSyncer struct { + mu sync.Mutex + Holder *Holder Node *Node @@ -588,6 +590,8 @@ func (s *HolderSyncer) IsClosing() bool { // SyncHolder compares the holder on host with the local holder and resolves differences. func (s *HolderSyncer) SyncHolder() error { + s.mu.Lock() // only allow one instance of SyncHolder to be running at a time + defer s.mu.Unlock() ti := time.Now() // Iterate over schema in sorted order. for _, di := range s.Holder.Schema() { diff --git a/holder_test.go b/holder_test.go index f758798a4..fe6a1a9f8 100644 --- a/holder_test.go +++ b/holder_test.go @@ -24,8 +24,6 @@ import ( "testing" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/http" - "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" ) @@ -350,49 +348,40 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { - t.Skip() // Until test.NewServer() works - - s := test.NewServer() - defer s.Close() - - uri, err := pilosa.NewURIFromAddress(s.URL) + c := test.MustNewCluster(t, 2) + c[0].Config.Cluster.ReplicaN = 2 + c[0].Config.AntiEntropy.Interval = 0 + c[1].Config.Cluster.ReplicaN = 2 + c[1].Config.AntiEntropy.Interval = 0 + err := c.Start() if err != nil { - t.Fatal(err) + t.Fatalf("starting cluster: %v", err) + } + defer c.Close() + + _, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index i: %v", err) + } + _, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index y: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field f: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field f0: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + if err != nil { + t.Fatalf("creating field z in y: %v", err) } - cluster := test.NewCluster(2) - client := http.GetHTTPClient(nil) - httpClient := http.NewInternalClientFromURI(uri, client) - cluster.InternalClient = httpClient - - // Create a local holder. - hldr0 := test.MustOpenHolder() - defer hldr0.Close() - - // Create a remote holder wrapped by an HTTP - hldr1 := test.MustOpenHolder() - defer hldr1.Close() - s.Handler.API.Holder = hldr1.Holder - s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr1.Holder - e.Node = cluster.Nodes[1] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) - } - - // Mock 2-node, fully replicated cluster. - cluster.ReplicaN = 2 - - cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) - cluster.Nodes[1].URI = *uri - - // Create fields on nodes. - for _, hldr := range []*test.Holder{hldr0, hldr1} { - hldr.MustCreateFieldIfNotExists("i", "f") - hldr.MustCreateFieldIfNotExists("i", "f0") - hldr.MustCreateFieldIfNotExists("y", "z") - } + hldr0 := &test.Holder{Holder: c[0].Server.Holder()} + hldr1 := &test.Holder{Holder: c[1].Server.Holder()} // Set data on the local holder. hldr0.SetBit("i", "f", 0, 10) @@ -414,42 +403,39 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr1.SetBit("y", "z", 10, (3*ShardWidth)+5) hldr1.SetBit("y", "z", 10, (3*ShardWidth)+7) - // Set highest shard. - hldr0.Index("i").SetRemoteMaxShard(1) - hldr0.Index("y").SetRemoteMaxShard(3) - - // Set up syncer. - syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - Node: cluster.Nodes[0], - Cluster: cluster, - Stats: pilosa.NopStatsClient, + err = c[0].Server.SyncData() + if err != nil { + t.Fatalf("syncing node 0: %v", err) } - - if err := syncer.SyncHolder(); err != nil { - t.Fatal(err) + err = c[1].Server.SyncData() + if err != nil { + t.Fatalf("syncing node 1: %v", err) } // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected columns(%d/200): %+v", i, a) + t.Errorf("unexpected columns(%d/0): %+v", i, a) + } + if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Errorf("unexpected columns(%d/2): %+v", i, a) + } + if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Errorf("unexpected columns(%d/3): %+v", i, a) + } + if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Errorf("unexpected columns(%d/120): %+v", i, a) + } + if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Errorf("unexpected columns(%d/200): %+v", i, a) } if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{ShardWidth + 5}) { - t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) + t.Errorf("unexpected columns(%d/d/f0): %+v", i, a) } if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * ShardWidth) + 4, (3 * ShardWidth) + 5, (3 * ShardWidth) + 7}) { - t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) + t.Errorf("unexpected columns(%d/y/z): %+v", i, a) } } } diff --git a/http/client.go b/http/client.go index 59efb4f72..345c572c1 100644 --- a/http/client.go +++ b/http/client.go @@ -717,6 +717,9 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in // BlockData returns row/column id pairs for a block. func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, block int) ([]uint64, []uint64, error) { + if uri == nil { + panic("need to pass a URI to BlockData") + } buf, err := proto.Marshal(&internal.BlockDataRequest{ Index: index, Field: field, @@ -727,7 +730,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, return nil, nil, errors.Wrap(err, "marshaling") } - u := uriPathToURL(c.defaultURI, "/fragment/block/data") + u := uriPathToURL(uri, "/fragment/block/data") req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) if err != nil { return nil, nil, errors.Wrap(err, "creating request") diff --git a/http/client_test.go b/http/client_test.go index e5a595605..fc8e896b5 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -26,23 +26,10 @@ import ( "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) -func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { - numNodes := len(c.Nodes) - hldr := make([]*test.Holder, numNodes) - server := make([]*test.Server, numNodes) - for i := 0; i < numNodes; i++ { - hldr[i] = test.MustOpenHolder() - server[i] = test.NewServer() - server[i].Handler.API.Cluster = c - server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() - server[i].Handler.API.Holder = hldr[i].Holder - } - return server, hldr -} - var defaultClient *gohttp.Client func init() { @@ -52,39 +39,19 @@ func init() { // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { - t.Skip() // Until test.NewServer() works + c := test.MustRunCluster(t, 3, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node0"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + []server.CommandOption{ + server.OptCommandServerOptions(pilosa.OptServerNodeID("node2"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, + ) + defer c.Close() - cluster := test.NewCluster(3) - s, hldr := createCluster(cluster) - - for i := 0; i < len(cluster.Nodes); i++ { - defer hldr[i].Close() - defer s[i].Close() - } - - s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr[0].Holder - e.Node = cluster.Nodes[0] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) - } - s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr[1].Holder - e.Node = cluster.Nodes[1] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) - } - s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, shards []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) - e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) - e.Holder = hldr[2].Holder - e.Node = cluster.Nodes[2] - e.Cluster = cluster - return e.Execute(ctx, index, query, shards, opt) + hldr := []test.Holder{} + for _, command := range c { + hldr = append(hldr, test.Holder{Holder: command.Server.Holder()}) } // Create a dispersed set of bitmaps across 3 nodes such that each individual node and shard width increment would reveal a different TopN. @@ -106,7 +73,7 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, s[i].Host(), owns) + t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c[i].URL(), owns) } } @@ -120,13 +87,21 @@ func TestClient_MultiNode(t *testing.T) { maxShard = x } } + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } hldr[0].MustSetBits("i", "f", 100, baseBit0+10) hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12) hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2, baseBit0+10) + hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2) hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) @@ -145,39 +120,27 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[0]).RecalculateCache() - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[1]).RecalculateCache() - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, shardNums[2]).RecalculateCache() + c[0].RecalculateCaches() + c[1].RecalculateCaches() + c[2].RecalculateCaches() // Connect to each node to compare results. client := make([]*Client, 3) - client[0] = MustNewClient(s[0].Host(), defaultClient) - client[1] = MustNewClient(s[1].Host(), defaultClient) - client[2] = MustNewClient(s[2].Host(), defaultClient) + client[0] = MustNewClient(c[0].URL(), defaultClient) + client[1] = MustNewClient(c[1].URL(), defaultClient) + client[2] = MustNewClient(c[2].URL(), defaultClient) topN := 4 queryRequest := &internal.QueryRequest{ Query: fmt.Sprintf(`TopN(f, n=%d)`, topN), Remote: false, } + result, err := client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) } - // Check the results before every node has the correct max shard value. - pairs := result.Results[0].Pairs - for _, pair := range pairs { - if pair.ID == 22 && pair.Count != 3 { - t.Fatalf("Invalid Cluster wide MaxShard prevents accurate calculation of %s", pair) - } - } - - // Set max shard to correct value. - hldr[0].Index("i").SetRemoteMaxShard(maxShard) - hldr[1].Index("i").SetRemoteMaxShard(maxShard) - hldr[2].Index("i").SetRemoteMaxShard(maxShard) - result, err = client[0].Query(context.Background(), "i", queryRequest) if err != nil { t.Fatal(err) @@ -189,7 +152,7 @@ func TestClient_MultiNode(t *testing.T) { } p := []*internal.Pair{ {ID: 100, Count: 12}, - {ID: 22, Count: 11}, + {ID: 22, Count: 10}, {ID: 98, Count: 8}, {ID: 99, Count: 7}} diff --git a/server.go b/server.go index 457a9123a..d64b9efa7 100644 --- a/server.go +++ b/server.go @@ -70,6 +70,7 @@ type Server struct { diagnosticInterval time.Duration maxWritesPerRequest int isCoordinator bool + syncer HolderSyncer primaryTranslateStore TranslateStore @@ -209,6 +210,20 @@ func OptServerIsCoordinator(is bool) ServerOption { } } +func OptServerNodeID(nodeID string) ServerOption { + return func(s *Server) error { + s.nodeID = nodeID + return nil + } +} + +func OptServerClusterHasher(h Hasher) ServerOption { + return func(s *Server) error { + s.cluster.Hasher = h + return nil + } +} + // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ @@ -328,6 +343,12 @@ func (s *Server) Open() error { // buffered channel. s.cluster.listenForJoins() + s.syncer.Holder = s.holder + s.syncer.Node = s.cluster.Node + s.syncer.Cluster = s.cluster + s.syncer.Closing = s.closing + s.syncer.Stats = s.holder.Stats.WithTags("HolderSyncer") + // Start background monitoring. s.wg.Add(3) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() @@ -370,12 +391,22 @@ func (s *Server) loadNodeID() string { return nodeID } +// SyncData manually invokes the anti entropy process which makes sure that this +// node has the data from all replicas across the cluster. +func (s *Server) SyncData() error { + return errors.Wrap(s.syncer.SyncHolder(), "syncing holder") +} + func (s *Server) monitorAntiEntropy() { + if s.antiEntropyInterval == 0 { + return // anti entropy disabled + } ticker := time.NewTicker(s.antiEntropyInterval) defer ticker.Stop() s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval) + // Initialize syncer with local holder and remote client. for { // Wait for tick or a close. select { @@ -385,18 +416,10 @@ func (s *Server) monitorAntiEntropy() { s.holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.logger.Printf("holder sync beginning") - - // Initialize syncer with local holder and remote client. - var syncer HolderSyncer - syncer.Holder = s.holder - syncer.Node = s.cluster.Node - syncer.Cluster = s.cluster - syncer.Closing = s.closing - syncer.Stats = s.holder.Stats.WithTags("HolderSyncer") // Sync holders. - if err := syncer.SyncHolder(); err != nil { + s.logger.Printf("holder sync beginning") + if err := s.syncer.SyncHolder(); err != nil { s.logger.Printf("holder sync error: err=%s", err) continue } diff --git a/server_test.go b/server_test.go deleted file mode 100644 index 04f8a6171..000000000 --- a/server_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "context" - "testing" - "time" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/server" - "github.com/pilosa/pilosa/test" -) - -// TestMonitorAntiEntropy is a regression test which which caught a bug where -// pilosa.Server was not having its remoteClient field set by an option and so -// it was using a nil client in monitorAntiEntropy. -func TestMonitorAntiEntropy(t *testing.T) { - cluster := test.MustRunCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)}) - client := cluster[1].Client() - err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - - err = client.CreateField(context.Background(), "balh", "fralh") - if err != nil { - t.Fatalf("creating field: %v", err) - } - - time.Sleep(time.Millisecond * 40) - for _, m := range cluster { - err := m.Close() - if err != nil { - t.Fatal(err) - } - } - -} diff --git a/test/cluster.go b/test/cluster.go index d1ee6af80..d7363e320 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -21,6 +21,11 @@ import ( "github.com/pilosa/pilosa" ) +// modHasher represents a simple, mod-based hashing. +type ModHasher struct{} + +func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } + // NewCluster returns a cluster with n nodes and uses a mod-based hasher. func NewCluster(n int) *pilosa.Cluster { path, err := ioutil.TempDir("", "pilosa-cluster-") @@ -30,7 +35,7 @@ func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() c.ReplicaN = 1 - c.Hasher = newModHasher() + c.Hasher = &ModHasher{} c.Path = path c.Topology = pilosa.NewTopology() @@ -48,14 +53,6 @@ func NewCluster(n int) *pilosa.Cluster { return c } -// modHasher represents a simple, mod-based hashing. -type modHasher struct{} - -// newModHasher returns a new instance of ModHasher with n buckets. -func newModHasher() *modHasher { return &modHasher{} } - -func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } - // newURI is a test URI creator that intentionally swallows errors. func newURI(scheme, host string, port uint16) pilosa.URI { uri := pilosa.DefaultURI() diff --git a/test/holder.go b/test/holder.go index ff308062c..6cdd57f55 100644 --- a/test/holder.go +++ b/test/holder.go @@ -121,6 +121,15 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { return row } +func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + return f.RowAttrStore() +} + // ViewRow returns a Row for a given field and view. func (h *Holder) ViewRow(index, field, view string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) @@ -142,7 +151,10 @@ func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { if err != nil { panic(err) } - f.SetBit(rowID, columnID, nil) + _, err = f.SetBit(rowID, columnID, nil) + if err != nil { + panic(err) + } } // ClearBit clears a bit on the given field. @@ -152,7 +164,10 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { if err != nil { panic(err) } - f.ClearBit(rowID, columnID) + _, err = f.ClearBit(rowID, columnID) + if err != nil { + panic(err) + } } // MustSetBits sets columns on a row. Panic on error.