diff --git a/executor_test.go b/executor_test.go index b66fcf81e..f2aab6fff 100644 --- a/executor_test.go +++ b/executor_test.go @@ -35,106 +35,76 @@ import ( // Ensure a row query can be executed. func TestExecutor_Execute_Row(t *testing.T) { - t.Run("Row", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := `` + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20) + + `SetRowAttrs(f, 10, foo="bar", baz=123)` + + `Set(1000, f=100)` + + `SetColumnAttrs(1000, foo="bar", baz=123)` + readQueries := []string{ + `Row(f=10)`, + `Options(Row(f=10), excludeColumns=true)`, + `Options(Row(f=10), excludeRowAttrs=true)`, } - if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit column attributes. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeColumns: true}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + if columns := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + } else if attrs := responses[1].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } // Inhibit row attributes. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`, ExcludeRowAttrs: true}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := responses[2].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) - t.Run("Column", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) - } - if err := index.ColumnAttrStore().SetAttrs(ShardWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { - t.Fatal(err) + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one-hundred", f=1) + Set("two-hundred", f=1)` + readQueries := []string{`Row(f=1)`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred", "two-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) } }) - t.Run("Keys", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) - if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { - t.Fatal(err) + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(100, f="one") + Set(200, f="one")` + readQueries := []string{`Row(f="one")`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100, 200}) { + t.Fatalf("unexpected columns: %+v", columns) } + }) - _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ - Index: "i", - Query: `` + - `Set("foo", f="bar")` + "\n" + - `Set("foo", f="baz")` + "\n" + - `Set("bat", f="bar")` + "\n" + - `Set("aaa", f="bbb")` + "\n", - }) - if err != nil { - t.Fatalf("querying: %v", err) - } - - if results, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ - Index: "i", - Query: `Row(f="bar")`, - }); err != nil { - t.Fatal(err) - } else if diff := cmp.Diff(results.Results, []interface{}{ + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := `` + + `Set("foo", f="bar")` + "\n" + + `Set("foo", f="baz")` + "\n" + + `Set("bat", f="bar")` + "\n" + + `Set("aaa", f="bbb")` + "\n" + readQueries := []string{`Row(f="bar")`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if diff := cmp.Diff(responses[0].Results, []interface{}{ &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { t.Fatal(diff) @@ -144,20 +114,68 @@ func TestExecutor_Execute_Row(t *testing.T) { // Ensure a difference query can be executed. func TestExecutor_Execute_Difference(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 1) - hldr.SetBit("i", "general", 10, 2) - hldr.SetBit("i", "general", 10, 3) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, 4) + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, 2) + hldr.SetBit("i", "general", 10, 3) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, 4) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("two", f=10) + Set("three", f=10) + Set("two", f=11) + Set("four", f=11)` + readQueries := []string{`Difference(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "three"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(2, f="ten") + Set(3, f="ten") + Set(2, f="eleven") + Set(4, f="eleven")` + readQueries := []string{`Difference(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("two", f="ten") + Set("three", f="ten") + Set("two", f="eleven") + Set("four", f="eleven")` + readQueries := []string{`Difference(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "three"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure an empty difference query behaves properly. @@ -174,22 +192,73 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { // Ensure an intersect query can be executed. func TestExecutor_Execute_Intersect(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 1) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 10, ShardWidth+2) + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.SetBit("i", "general", 11, 1) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 11, 1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=10) + Set("one", f=11) + Set("two", f=11) + Set("two-hundred", f=11)` + readQueries := []string{`Intersect(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "two-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="ten") + Set(1, f="eleven") + Set(2, f="eleven") + Set(200, f="eleven")` + readQueries := []string{`Intersect(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 200}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="ten") + Set("one", f="eleven") + Set("two", f="eleven") + Set("two-hundred", f="eleven")` + readQueries := []string{`Intersect(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "two-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure an empty intersect query behaves properly. @@ -204,21 +273,72 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { // Ensure a union query can be executed. func TestExecutor_Execute_Union(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 10, ShardWidth+2) + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=10) + Set("one", f=11) + Set("two", f=11) + Set("two-hundred", f=11)` + readQueries := []string{`Union(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "one-hundred", "two-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="ten") + Set(1, f="eleven") + Set(2, f="eleven") + Set(200, f="eleven")` + readQueries := []string{`Union(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 100, 200}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="ten") + Set("one", f="eleven") + Set("two", f="eleven") + Set("two-hundred", f="eleven")` + readQueries := []string{`Union(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one", "one-hundred", "two-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure an empty union query behaves properly. @@ -237,44 +357,138 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { // Ensure a xor query can be executed. func TestExecutor_Execute_Xor(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 10, ShardWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 10, ShardWidth+2) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) - } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=10) + Set("one", f=11) + Set("two", f=11) + Set("two-hundred", f=11)` + readQueries := []string{`Xor(Row(f=10), Row(f=11))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="ten") + Set(1, f="eleven") + Set(2, f="eleven") + Set(200, f="eleven")` + readQueries := []string{`Xor(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 100}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="ten") + Set("one", f="eleven") + Set("two", f="eleven") + Set("two-hundred", f="eleven")` + readQueries := []string{`Xor(Row(f="ten"), Row(f="eleven"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred", "two"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("RowIDColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "f", 10, 3) - hldr.SetBit("i", "f", 10, ShardWidth+1) - hldr.SetBit("i", "f", 10, ShardWidth+2) + hldr.SetBit("i", "f", 10, 3) + hldr.SetBit("i", "f", 10, ShardWidth+1) + hldr.SetBit("i", "f", 10, ShardWidth+2) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { + t.Fatal(err) + } else if res.Results[0] != uint64(3) { + t.Fatalf("unexpected n: %d", res.Results[0]) + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("three", f=10) + Set("one-hundred", f=10) + Set("two-hundred", f=11)` + readQueries := []string{`Count(Row(f=10))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if responses[0].Results[0] != uint64(2) { + t.Fatalf("unexpected n: %d", responses[0].Results[0]) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(1, f="ten") + Set(100, f="ten") + Set(200, f="eleven")` + readQueries := []string{`Count(Row(f="ten"))`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if responses[0].Results[0] != uint64(2) { + t.Fatalf("unexpected n: %d", responses[0].Results[0]) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("one", f="ten") + Set("one-hundred", f="ten") + Set("two-hundred", f="eleven")` + readQueries := []string{`Count(Row(f="ten"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if responses[0].Results[0] != uint64(2) { + t.Fatalf("unexpected n: %d", responses[0].Results[0]) + } + }) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { - t.Fatal(err) - } else if res.Results[0] != uint64(3) { - t.Fatalf("unexpected n: %d", res.Results[0]) - } } // Ensure a set query can be executed. -func TestExecutor_Execute_SetBit(t *testing.T) { - t.Run("ID", func(t *testing.T) { +func TestExecutor_Execute_Set(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -319,7 +533,25 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) }) - t.Run("Keys", func(t *testing.T) { + t.Run("RowIDColumnKey", func(t *testing.T) { + readQueries := []string{`Set("three", f=10)`} + responses := runCallTest(t, "", readQueries, + &pilosa.IndexOptions{Keys: true}) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + readQueries := []string{`Set(1, f="ten")`} + responses := runCallTest(t, "", readQueries, + nil, pilosa.OptFieldKeys()) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { cmd := test.MustRunCluster(t, 1)[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -376,6 +608,49 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) } +// Ensure a set query can be executed. +func TestExecutor_Execute_Clear(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := `Set(3, f=10)` + readQueries := []string{`Clear(3, f=10)`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := `Set("three", f=10)` + readQueries := []string{`Clear("three", f=10)`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := `Set(1, f="ten")` + readQueries := []string{`Clear(1, f="ten")`} + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldKeys()) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := `Set("one", f="ten")` + readQueries := []string{`Clear("one", f="ten")`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) + if !responses[0].Results[0].(bool) { + t.Fatalf("expected column changed") + } + }) +} + // Ensure a set query can be executed on a bool field. func TestExecutor_Execute_SetBool(t *testing.T) { t.Run("Basic", func(t *testing.T) { @@ -594,7 +869,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - t.Run("ID", func(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -635,7 +910,89 @@ func TestExecutor_Execute_TopN(t *testing.T) { } }) - t.Run("Keys", func(t *testing.T) { + t.Run("RowIDColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f"); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other"); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", f=0) + Set("one", f=0) + Set("sw", f=0) + Set("sw2", f=0) + Set("sw3", f=0) + Set("zero", f=10) + Set("sw", f=10) + Set("sw", f=20) + Set("zero", other=0) + `}); err != nil { + t.Fatal(err) + } + + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", f="zero") + Set("one", f="zero") + Set("sw", f="zero") + Set("sw2", f="zero") + Set("sw3", f="zero") + Set("zero", f="ten") + Set("sw", f="ten") + Set("sw", f="twenty") + Set("zero", other="zero") + `}); err != nil { + t.Fatal(err) + } + + err := c[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ + {Key: "zero", Count: 5}, + {Key: "ten", Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -820,255 +1177,443 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // Ensure Min() and Max() queries can be executed. func TestExecutor_Execute_MinMax(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { - t.Fatal(err) - } - - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(0, x=0) - Set(3, x=0) - Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) - Set(1, x=1) - Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) - - Set(0, f=20) - Set(1, f=-5) - Set(2, f=-5) - Set(3, f=10) - Set(` + strconv.Itoa(ShardWidth) + `, f=30) - Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) - Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) - Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) - `}); err != nil { - t.Fatal(err) - } - - t.Run("Min", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: -5, cnt: 2}, - {filter: `Row(x=0)`, exp: 10, cnt: 1}, - {filter: `Row(x=1)`, exp: -5, cnt: 1}, - {filter: `Row(x=2)`, exp: 40, cnt: 1}, + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Min(field=f)` - } else { - pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, x=0) + Set(3, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) + Set(1, x=1) + Set(` + strconv.Itoa(ShardWidth+2) + `, x=2) + + Set(0, f=20) + Set(1, f=-5) + Set(2, f=-5) + Set(3, f=10) + Set(` + strconv.Itoa(ShardWidth) + `, f=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, f=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=60) + `}); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Row(x=0)`, exp: 10, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(field=f)` + } else { + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Row(x=0)`, exp: 60, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(field=f)` + } else { + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) }) - t.Run("Max", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: 60, cnt: 1}, - {filter: `Row(x=0)`, exp: 60, cnt: 1}, - {filter: `Row(x=1)`, exp: -5, cnt: 1}, - {filter: `Row(x=2)`, exp: 40, cnt: 1}, + t.Run("ColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}) + if err != nil { + t.Fatal(err) } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Max(field=f)` - } else { - pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", x=0) + Set("three", x=0) + Set("sw1", x=0) + Set("one", x=1) + Set("sw2", x=2) + + Set("zero", f=20) + Set("one", f=-5) + Set("two", f=-5) + Set("three", f=10) + Set("sw", f=30) + Set("sw2", f=40) + Set("sw3", f=50) + Set("sw1", f=60) + `}); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Row(x=0)`, exp: 10, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(field=f)` + } else { + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Row(x=0)`, exp: 60, cnt: 1}, + {filter: `Row(x=1)`, exp: -5, cnt: 1}, + {filter: `Row(x=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(field=f)` + } else { + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) }) } // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { - t.Fatal(err) - } - - if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { - t.Fatal(err) - } - - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(0, x=0) - Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) - - Set(0, foo=20) - Set(0, bar=2000) - Set(` + strconv.Itoa(ShardWidth) + `, foo=30) - Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) - Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) - Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) - Set(0, other=1000) - `}); err != nil { - t.Fatal(err) - } - - t.Run("NoFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, x=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) + + Set(0, foo=20) + Set(0, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=40) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=50) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + `}); err != nil { + t.Fatal(err) + } + + t.Run("NoFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("WithFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) }) - t.Run("WithFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + t.Run("ColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}) + if err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("zero", x=0) + Set("sw1", x=0) + + Set("zero", foo=20) + Set("zero", bar=2000) + Set("sw", foo=30) + Set("sw2", foo=40) + Set("sw3", foo=50) + Set("sw1", foo=60) + Set("zero", other=1000) + `}); err != nil { + t.Fatal(err) + } + + t.Run("NoFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("WithFilter", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) }) } // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) - // Create index. - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - - // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { - t.Fatal(err) - } - - // Set columns. - cc := ` - Set(2, f=1, 1999-12-31T00:00) - Set(3, f=1, 2000-01-01T00:00) - Set(4, f=1, 2000-01-02T00:00) - Set(5, f=1, 2000-02-01T00:00) - Set(6, f=1, 2001-01-01T00:00) - Set(7, f=1, 2002-01-01T02:00) - - Set(2, f=1, 1999-12-30T00:00) - Set(2, f=1, 2002-02-01T00:00) - Set(2, f=10, 2001-01-01T00:00) - ` - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { - t.Fatal(err) - } - - t.Run("Standard", func(t *testing.T) { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { - t.Fatalf("unexpected columns: %+v", columns) + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear( 2, f=1)`, + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, } + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) }) - t.Run("Clear", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f=1)`}); err != nil { - t.Fatal(err) + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f=1, 1999-12-31T00:00) + Set("three", f=1, 2000-01-01T00:00) + Set("four", f=1, 2000-01-02T00:00) + Set("five", f=1, 2000-02-01T00:00) + Set("six", f=1, 2001-01-01T00:00) + Set("seven", f=1, 2002-01-01T02:00) + + Set("two", f=1, 1999-12-30T00:00) + Set("two", f=1, 2002-02-01T00:00) + Set("two", f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear("two", f=1)`, + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { - t.Fatalf("unexpected columns: %+v", columns) - } - }) -} + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) -// Ensure a range query with keys can be executed. -func TestExecutor_Execute_Range_WithKeys(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Create index. - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - - // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), pilosa.OptFieldKeys()); err != nil { - t.Fatal(err) - } - - // Set columns. - cc := ` - Set(2, f="foo", 1999-12-31T00:00) - Set(3, f="foo", 2000-01-01T00:00) - Set(4, f="foo", 2000-01-02T00:00) - Set(5, f="foo", 2000-02-01T00:00) - Set(6, f="foo", 2001-01-01T00:00) - Set(7, f="foo", 2002-01-01T02:00) - - Set(2, f="foo", 1999-12-30T00:00) - Set(2, f="foo", 2002-02-01T00:00) - Set(2, f="bar", 2001-01-01T00:00) - ` - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { - t.Fatal(err) - } - - t.Run("Standard", func(t *testing.T) { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { - t.Fatalf("unexpected columns: %+v", columns) - } + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) }) - t.Run("Clear", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f="foo")`}); err != nil { - t.Fatal(err) - } + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f="foo", 1999-12-31T00:00) + Set(3, f="foo", 2000-01-01T00:00) + Set(4, f="foo", 2000-01-02T00:00) + Set(5, f="foo", 2000-02-01T00:00) + Set(6, f="foo", 2001-01-01T00:00) + Set(7, f="foo", 2002-01-01T02:00) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { - t.Fatalf("unexpected columns: %+v", columns) + Set(2, f="foo", 1999-12-30T00:00) + Set(2, f="foo", 2002-02-01T00:00) + Set(2, f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear( 2, f="foo")`, + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, } + responses := runCallTest(t, writeQuery, readQueries, + nil, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f="foo", 1999-12-31T00:00) + Set("three", f="foo", 2000-01-01T00:00) + Set("four", f="foo", 2000-01-02T00:00) + Set("five", f="foo", 2000-02-01T00:00) + Set("six", f="foo", 2001-01-01T00:00) + Set("seven", f="foo", 2002-01-01T02:00) + + Set("two", f="foo", 1999-12-30T00:00) + Set("two", f="foo", 2002-02-01T00:00) + Set("two", f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Clear("two", f="foo")`, + `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) }) } @@ -1478,174 +2023,98 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { } -func TestExecutor_QueryCall(t *testing.T) { +func TestExecutor_ExecuteOptions(t *testing.T) { t.Run("excludeRowAttrs", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), excludeRowAttrs=true)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + writeQuery := ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar")` + readQueries := []string{`Options(Row(f=10), excludeRowAttrs=true)`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("excludeColumns", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), excludeColumns=true)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + writeQuery := ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar")` + readQueries := []string{`Options(Row(f=10), excludeColumns=true)`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("columnAttrs", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + writeQuery := ` Set(100, f=10) - SetColumnAttrs(100, foo="bar") - `}); err != nil { - t.Fatal(err) - } - + SetColumnAttrs(100, foo="bar")` + readQueries := []string{`Options(Row(f=10), columnAttrs=true)`} + responses := runCallTest(t, writeQuery, readQueries, nil) targetColAttrSets := []*pilosa.ColumnAttrSet{ {ID: 100, Attrs: map[string]interface{}{"foo": "bar"}}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), columnAttrs=true)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { + } else if attrs := responses[0].ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("columnAttrsWithKeys", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldKeys()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + writeQuery := ` Set("one-hundred", f="ten") - SetColumnAttrs("one-hundred", foo="bar") - `}); err != nil { - t.Fatal(err) - } + SetColumnAttrs("one-hundred", foo="bar")` + readQueries := []string{`Options(Row(f="ten"), columnAttrs=true)`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldKeys()) targetColAttrSets := []*pilosa.ColumnAttrSet{ {Key: "one-hundred", Attrs: map[string]interface{}{"foo": "bar"}}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f="ten"), columnAttrs=true)`}); err != nil { - t.Fatal(err) - } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) { t.Fatalf("unexpected keys: %+v", keys) - } else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { + } else if attrs := responses[0].ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) t.Run("shards", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` - Set(100, f=10) - Set(%d, f=10) - Set(%d, f=10) - `, ShardWidth, ShardWidth*2)}); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f=10), shards=[0, 2])`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100, ShardWidth * 2}) { + writeQuery := fmt.Sprintf(` + Set(100, f=10) + Set(%d, f=10) + Set(%d, f=10)`, ShardWidth, ShardWidth*2) + readQueries := []string{`Options(Row(f=10), shards=[0, 2])`} + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100, ShardWidth * 2}) { t.Fatalf("unexpected columns: %+v", bits) } }) t.Run("multipleOpt", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - - // Set columns for rows 0, 10, & 20 across two shards. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` - Set(100, f=10) - SetRowAttrs(f, 10, foo="bar") - `}); err != nil { - t.Fatal(err) + writeQuery := ` + Set(100, f=10) + SetRowAttrs(f, 10, foo="bar")` + readQueries := []string{ + `Options(Row(f=10), excludeColumns=true) + Options(Row(f=10), excludeRowAttrs=true)`, } - - req := &pilosa.QueryRequest{ - Index: "i", - Query: `Options(Row(f=10), excludeColumns=true)Options(Row(f=10), excludeRowAttrs=true)`, - } - if res, err := c[0].API.Query(context.Background(), req); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + responses := runCallTest(t, writeQuery, readQueries, nil) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { + } else if attrs := responses[0].Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar"}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) - } else if bits := res.Results[1].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { + } else if bits := responses[0].Results[1].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{100}) { t.Fatalf("unexpected columns: %+v", bits) - } else if attrs := res.Results[1].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + } else if attrs := responses[0].Results[1].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } }) @@ -1699,170 +2168,157 @@ func TestExecutor_Execute_Existence(t *testing.T) { // Ensure a not query can be executed. func TestExecutor_Execute_Not(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20) + readQueries := []string{ + `Not(Row(f=20))`, + `Not(Row(f=0))`, + `Not(Union(Row(f=10), Row(f=20)))`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}) - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20), - }); err != nil { - t.Fatal(err) - } + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } - // Populated row. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=20))`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { - t.Fatalf("unexpected columns: %+v", bits) - } + if bits := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", bits) + } - // Populated row. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=0))`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected columns: %+v", bits) - } + if bits := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) - // All existing. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Union(Row(f=10), Row(f=20)))`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { - t.Fatalf("unexpected columns: %+v", bits) - } + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("three", f=10) + Set("sw1", f=10) + Set("sw2", f=20)` + readQueries := []string{`Not(Row(f=20))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{ + TrackExistence: true, + Keys: true, + }) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "sw1"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := fmt.Sprintf(` + Set(3, f="ten") + Set(%d, f="ten") + Set(%d, f="twenty")`, ShardWidth+1, ShardWidth+2) + readQueries := []string{`Not(Row(f="twenty"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}, + pilosa.OptFieldKeys(), + ) + if cols := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, []uint64{3, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", cols) + } + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("three", f="ten") + Set("sw1", f="ten") + Set("sw2", f="twenty")` + readQueries := []string{`Not(Row(f="twenty"))`} + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{ + TrackExistence: true, + Keys: true, + }, pilosa.OptFieldKeys()) + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "sw1"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) } // Ensure a row can be cleared. func TestExecutor_Execute_ClearRow(t *testing.T) { + // Set and Mutex tests use the same data and queries + writeQuery := `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20) + + readQueries := []string{ + `Row(f=10)`, + `ClearRow(f=10)`, + `ClearRow(f=10)`, + `Row(f=10)`, + `Row(f=20)`, + } + t.Run("Set", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Clear the row and ensure we get a `true` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); !res { + if res := responses[1].Results[0].(bool); !res { t.Fatalf("unexpected clear row result: %+v", res) } // Clear the row again and ensure we get a `false` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); res { + if res := responses[2].Results[0].(bool); res { t.Fatalf("unexpected clear row result: %+v", res) } // Ensure the row is empty. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + if bits := responses[3].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) } // Ensure other rows were not affected. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + if bits := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } }) + t.Run("Mutex", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeMutex("none", 0)) - if err != nil { - t.Fatal(err) - } - - // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + - fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + - fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + - fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), - }); err != nil { - t.Fatal(err) - } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) { + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}, + pilosa.OptFieldTypeMutex("none", 0)) + if bits := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Clear the row and ensure we get a `true` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); !res { + if res := responses[1].Results[0].(bool); !res { t.Fatalf("unexpected clear row result: %+v", res) } // Clear the row again and ensure we get a `false` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); res { + if res := responses[2].Results[0].(bool); res { t.Fatalf("unexpected clear row result: %+v", res) } // Ensure the row is empty. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + if bits := responses[3].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) } // Ensure other rows were not affected. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { - t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + if bits := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } }) - t.Run("Time", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) - if err != nil { - t.Fatal(err) - } - // Set columns. - cc := ` + t.Run("Time", func(t *testing.T) { + writeQuery := ` Set(2, f=1, 1999-12-31T00:00) Set(3, f=1, 2000-01-01T00:00) Set(4, f=1, 2000-01-02T00:00) @@ -1872,42 +2328,36 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { Set(2, f=1, 1999-12-30T00:00) Set(2, f=1, 2002-02-01T00:00) - Set(2, f=10, 2001-01-01T00:00) - ` - rangeCheckQuery1 := `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)` - rangeCheckQuery10 := `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)` - - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { - t.Fatal(err) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, + `ClearRow(f=1)`, + `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, + `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`, } - - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{TrackExistence: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } // Clear the row and ensure we get a `true` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err != nil { - t.Fatal(err) - } else if res := res.Results[0].(bool); !res { + if res := responses[1].Results[0].(bool); !res { t.Fatalf("unexpected clear row result: %+v", res) } // Ensure the row is empty. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) } // Ensure other rows were not affected. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery10}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { + if columns := responses[3].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { t.Fatalf("unexpected columns: %+v", columns) } }) + t.Run("Int", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -1923,6 +2373,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { t.Fatal("expected clear row to return an error") } }) + t.Run("TopN", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -2171,3 +2622,42 @@ func benchmarkExistence(nn bool, b *testing.B) { func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, b) } func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } + +func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOptions *pilosa.IndexOptions, fieldOption ...pilosa.FieldOption) []pilosa.QueryResponse { + if indexOptions == nil { + indexOptions = &pilosa.IndexOptions{} + } + + c := test.MustRunCluster(t, 1) + defer c.Close() + + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", *indexOptions) + _, err := index.CreateField("f", fieldOption...) + if err != nil { + t.Fatal(err) + } + if writeQuery != "" { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: writeQuery, + }); err != nil { + t.Fatal(err) + } + } + + responses := []pilosa.QueryResponse{} + for _, query := range readQueries { + res, err := c[0].API.Query(context.Background(), + &pilosa.QueryRequest{ + Index: "i", + Query: query, + }) + if err != nil { + t.Fatal(err) + } + responses = append(responses, res) + } + + return responses +} diff --git a/holder.go b/holder.go index 80b13e3b6..899c25944 100644 --- a/holder.go +++ b/holder.go @@ -101,6 +101,9 @@ func NewHolder() *Holder { // Open initializes the root data directory for the holder. func (h *Holder) Open() error { + // Reset closing in case Holder is being reopened. + h.closing = make(chan struct{}) + h.setFileLimit() h.Logger.Printf("open holder path: %s", h.Path) @@ -178,6 +181,9 @@ func (h *Holder) Close() error { } } + // Reset opened in case Holder needs to be reopened. + h.opened = make(chan struct{}) + return nil } diff --git a/holder_internal_test.go b/holder_internal_test.go index c4230e9aa..0e61b87a5 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -287,3 +287,13 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { } } } + +// Ensure holder can reopen. +func TestHolderCleaner_Reopen(t *testing.T) { + h := NewHolder() + h.Path = "path" + h.Open() + h.Close() + h.Open() + h.Close() +}