From 319456bbd73493f3a12d24e224d50ae083bc0e3a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 11 Sep 2018 12:06:03 -0500 Subject: [PATCH] implements Not() query --- docs/administration.md | 2 ++ docs/query-language.md | 36 +++++++++++++++++++++--- executor.go | 34 ++++++++++++++++++++++ executor_test.go | 42 ++++++++++++++++++++++++++++ internal/test/querygenerator.go | 20 +++++++++++++ internal/test/querygenerator_test.go | 32 +++++++++++++++++++++ 6 files changed, 162 insertions(+), 4 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 71ab54672..cd4e30184 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -294,6 +294,8 @@ We currently track the following events - **Union:** Count of Union queries. - **Intersection:** Count of Intersection queries. - **Difference:** Count of Difference queries. +- **Xor:** Count of Xor queries. +- **Not:** Count of Not queries. - **Count:** Count of Count queries. - **Range:** Count of Range queries. - **Snapshot:** Event count when the snapshot process is triggered. diff --git a/docs/query-language.md b/docs/query-language.md index a61a9bf09..d289fb9a2 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -48,7 +48,7 @@ curl localhost:10101/index/repository/query \ * `UINT` An unsigned integer (e.g. 42839) * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. -* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range` +* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`, `Not` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations @@ -94,7 +94,7 @@ Set(10, stargazer=1, 2016-01-01T00:00) Set multiple bits in a single request: ```request -Set(1, stargazer=10) Set(2, stargazer=10) Set(1, stargazer=20) Set(2, stargazer=30) +Set(10, stargazer=1) Set(20, stargazer=1) Set(10, stargazer=2) Set(30, stargazer=2) ``` ```response {"results":[false,true,true,true]} @@ -358,7 +358,7 @@ Difference(Row(stargazer=2), Row(stargazer=1)) {"attrs":{},"columns":[30]} ``` -* columnss are repositories that were starred by user 2 BUT NOT user 1 +* columns are repositories that were starred by user 2 BUT NOT user 1 #### Xor @@ -384,11 +384,39 @@ Query columns with a bit set in exactly one of two rows (repositories that are s Xor(Row(stargazer=2), Row(stargazer=1)) ``` ```response -{"results":[{"attrs":{},"columns":[10,20,30]}]} +{"results":[{"attrs":{},"columns":[20,30]}]} ``` * columns are repositories that were starred by user 1 XOR user 2 (user 1 or user 2, but not both) +#### Not + +**Spec:** + +``` +Not() +``` + +**Description:** + +Not returns the inverse of all of the bits from the `ROW_CALL` argument. The Not query requires that `trackExistence` has been enabled on the Index. + +**Result Type:** object with attrs and columns + +attrs will always be empty + +**Examples:** + +Query columns with a bit set in one row and not another (repositories that are starred by one user and not another): +```request +Not(Row(stargazer=1)) +``` +```response +{"results":[{"attrs":{},"columns":[30]}]} +``` + +* columns are repositories that were not starred by user 1 + #### Count **Spec:** diff --git a/executor.go b/executor.go index 389cb573b..b0083a514 100644 --- a/executor.go +++ b/executor.go @@ -443,6 +443,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeUnionShard(ctx, index, c, shard) case "Xor": return e.executeXorShard(ctx, index, c, shard) + case "Not": + return e.executeNotShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -1007,6 +1009,38 @@ func (e *executor) executeXorShard(ctx context.Context, index string, c *pql.Cal return other, nil } +// executeNotShard executes a not() call for a local shard. +func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + if len(c.Children) == 0 { + return nil, errors.New("Not() requires an input row") + } else if len(c.Children) > 1 { + return nil, errors.New("Not() only accepts a single row input") + } + + // Make sure the index supports existence tracking. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } else if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", index) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + existenceRow = NewRow() + } else { + existenceRow = existenceFrag.row(0) + } + + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return nil, err + } + + return existenceRow.Difference(row), nil +} + // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { if len(c.Children) == 0 { diff --git a/executor_test.go b/executor_test.go index 5704e607b..ee919f001 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1543,6 +1543,48 @@ func TestExecutor_Execute_Existence(t *testing.T) { }) } +// Ensure a not query can be executed. +func TestExecutor_Execute_Not(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20), + }); err != nil { + t.Fatal(err) + } + + // Populated row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=20))`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Populated row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=0))`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // All existing. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Union(Row(f=10), Row(f=20)))`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } +} + func benchmarkExistence(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() diff --git a/internal/test/querygenerator.go b/internal/test/querygenerator.go index a7cf70c9b..ea0dc81c0 100644 --- a/internal/test/querygenerator.go +++ b/internal/test/querygenerator.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -95,11 +109,17 @@ func Difference(args ...*pql.Call) *pql.Call { kvargs, children := magic(args) return &pql.Call{Name: "Difference", Args: kvargs, Children: children} } + func Xor(args ...*pql.Call) *pql.Call { kvargs, children := magic(args) return &pql.Call{Name: "Xor", Args: kvargs, Children: children} } +func Not(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Not", Args: kvargs, Children: children} +} + func Between(_ string, min, max int) *pql.Call { return &pql.Call{ Name: "Range", diff --git a/internal/test/querygenerator_test.go b/internal/test/querygenerator_test.go index 4ca50c265..9bb404ab4 100644 --- a/internal/test/querygenerator_test.go +++ b/internal/test/querygenerator_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -208,6 +222,24 @@ func TestPQL_Generator(t *testing.T) { }, }, }, + { + pql: `Not(Row(aaa=10))`, + calc: PQL(Not(Row("aaa", 10))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Not", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + }, + }, + }, + }, + }, } { if !Compare(u.calc, u.exp) {