From 6dc3837c9a2707330500f6979bcb04b8680bc07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Tue, 31 Mar 2020 16:29:14 +0200 Subject: [PATCH] WIP: fix 'unknown call: Distinct' error (#213) --- executor.go | 4 + executor_test.go | 102 +++++++++++ testdata/schema.json | 424 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 530 insertions(+) create mode 100644 testdata/schema.json diff --git a/executor.go b/executor.go index 7088d7450..482a10e6e 100644 --- a/executor.go +++ b/executor.go @@ -321,6 +321,10 @@ func (e *executor) handlePreCalls(ctx context.Context, index string, c *pql.Call // like Distinct, where you can't predict output shard for a result // from the shard being queried. if newIndex != "" && newIndex != index { + if err := e.handlePreCallChildren(ctx, index, c, shards, opt); err != nil { + return err + } + c.Type = pql.PrecallGlobal index = newIndex // we need to recompute shards, then diff --git a/executor_test.go b/executor_test.go index e052a7242..3e81bd156 100644 --- a/executor_test.go +++ b/executor_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "bytes" "context" "encoding/json" "flag" @@ -4980,3 +4981,104 @@ func TestExecutor_Execute_NoIndex(t *testing.T) { t.Fatal("expecting error: 'index systems does not exist'") } } + +func TestExecutor_Execute_CountDistinct(t *testing.T) { + data, err := ioutil.ReadFile("testdata/schema.json") + if err != nil { + t.Fatal(err) + } + + c := test.MustRunCluster(t, 1) + + defer c.Close() + api := c[0].API + + schema := &pilosa.Schema{} + if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil { + t.Fatal(err) + } + if err := api.ApplySchema(context.TODO(), schema, false); err != nil { + t.Fatal(err) + } + + writeQuery := `Set(100, type=AntidotePoint)Set(100, equip_id=100)Set(100, site_id=100)Set(100, id=100)` + for _, i := range schema.Indexes { + if _, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: i.Name, Query: writeQuery}); err != nil { + t.Fatal(err) + } + } + + // test query - Distinct of Distincts + pql := `Distinct( + Intersect( + Distinct( + Intersect(Row(type=AntidotePoint)), + index=power_ts, field=equip_id), + Distinct( + Intersect(Row(type=AntidotePoint)), + index=power_ts, field=equip_id) + ), index=equipment, field=site_id)` + + // Check if test query gives correct results (one column 100) + t.Run("Distinct", func(t *testing.T) { + resp, err := api.Query(context.TODO(), &pilosa.QueryRequest{ + Index: "sites", + Query: pql, + }) + if err != nil { + t.Fatal(err) + } + r, ok := resp.Results[0].(pilosa.SignedRow) + if !ok { + t.Fatalf("invalid response type, expected: pilosa.SignedRow, got: %T", resp.Results[0]) + } + if r.Pos.Count() != 1 { + t.Fatalf("invalid pilosa.SignedRow.Pos.Count, expected: 1, got: %v", r.Pos.Count()) + } + if r.Pos.Columns()[0] != 100 { + t.Fatalf("invalid pilosa.SignedRow.Pos.Columns, expected: [100], got: %v", r.Pos.Columns()) + } + }) + + // Following tests check if wrapping Distinct of Distincts query by Count and GroupBy + // is fixed and does not give an error: 'unknown call: Distinct' error. + + // Check if Count on test query gives correct, exactly 1 result + t.Run("Count(Distinct)", func(t *testing.T) { + resp, err := api.Query(context.TODO(), &pilosa.QueryRequest{ + Index: "sites", + Query: fmt.Sprintf("Count(%s)", pql), + }) + if err != nil { + t.Fatal(err) + } + cnt, ok := resp.Results[0].(uint64) + if !ok { + t.Fatalf("invalid response type, expected: uint64, got: %T", resp.Results[0]) + } + if cnt != 1 { + t.Fatalf("invalid result, expected: 1, got: %v", cnt) + } + }) + + // Check if GroupBy on test query gives correct, exactly 1 result + t.Run("GroupBy(Distinct)", func(t *testing.T) { + resp, err := api.Query(context.TODO(), &pilosa.QueryRequest{ + Index: "sites", + Query: fmt.Sprintf("GroupBy(Rows(type), filter=%s)", pql), + }) + if err != nil { + t.Fatal(err) + } + gc, ok := resp.Results[0].([]pilosa.GroupCount) + if !ok { + t.Fatalf("invalid response type, expected: []pilosa.GroupCount, got: %T", resp.Results[0]) + } + if len(gc) != 1 { + t.Fatalf("invalid group count length, expected: 1, got: %v", len(gc)) + } + if gc[0].Count != 1 { + t.Fatalf("invalid group count count, expected: 1, got: %v", gc[0].Count) + } + }) +} diff --git a/testdata/schema.json b/testdata/schema.json new file mode 100644 index 000000000..4bd36a3a9 --- /dev/null +++ b/testdata/schema.json @@ -0,0 +1,424 @@ +{ + "indexes": [ + { + "name": "equipment", + "options": { + "keys": true, + "trackExistence": true + }, + "fields": [ + { + "name": "classification", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "equip_id", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "id", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "last_inspection", + "options": { + "type": "int", + "base": 0, + "bitDepth": 15, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "latitude", + "options": { + "type": "decimal", + "base": 0, + "scale": 4, + "bitDepth": 19, + "min": -1000, + "max": 1000, + "keys": false + } + }, + { + "name": "longitude", + "options": { + "type": "decimal", + "base": 0, + "scale": 4, + "bitDepth": 21, + "min": -1000, + "max": 1000, + "keys": false + } + }, + { + "name": "maintainer", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "region", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "service_provider", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "site_id", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "timestamp", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "type", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + } + ], + "shardWidth": 1048576 + }, + { + "name": "power_ts", + "options": { + "keys": true, + "trackExistence": true + }, + "fields": [ + { + "name": "cost", + "options": { + "type": "int", + "base": 0, + "bitDepth": 20, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "date_acquired", + "options": { + "type": "int", + "base": 0, + "bitDepth": 15, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "domain", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "equip_id", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "id", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "last_maintenance", + "options": { + "type": "int", + "base": 0, + "bitDepth": 15, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "manufacturer", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "model", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "name", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "site_id", + "options": { + "type": "int", + "base": 0, + "bitDepth": 23, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "sites_id", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "timestamp", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "type", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + } + ], + "shardWidth": 1048576 + }, + { + "name": "sites", + "options": { + "keys": true, + "trackExistence": true + }, + "fields": [ + { + "name": "cost", + "options": { + "type": "int", + "base": 0, + "bitDepth": 20, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "date_acquired", + "options": { + "type": "int", + "base": 0, + "bitDepth": 15, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "domain", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "equip_id", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "id", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": false + } + }, + { + "name": "last_maintenance", + "options": { + "type": "int", + "base": 0, + "bitDepth": 15, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "manufacturer", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "model", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "name", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "site_id", + "options": { + "type": "int", + "base": 0, + "bitDepth": 23, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "timestamp", + "options": { + "type": "int", + "base": 0, + "bitDepth": 1, + "min": -1000, + "max": 1000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "type", + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + } + ], + "shardWidth": 1048576 + } + ] +}