From 9eb8fcb37fad987d3cf16e3a080090ec3b479dbf Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Mon, 6 Jul 2020 14:50:52 -0400 Subject: [PATCH 1/7] initial impl of like --- like.go | 159 ++++++++++++++++++++++++++++++++++++++++++++ like_test.go | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 like.go create mode 100644 like_test.go diff --git a/like.go b/like.go new file mode 100644 index 000000000..db92232d7 --- /dev/null +++ b/like.go @@ -0,0 +1,159 @@ +package pilosa + +import ( + "strings" + "unicode/utf8" +) + +func tokenizeLike(like string) []string { + var tokens []string + for like != "" { + var token string + i := strings.IndexAny(like, "%_") + switch { + case i == 0: + j := 1 + for j < len(like) && (like[j] == '%' || like[j] == '_') { + j++ + } + token, like = like[:j], like[j:] + case i < 0: + token, like = like, "" + default: + token, like = like[:i], like[i:] + } + tokens = append(tokens, token) + } + return tokens +} + +type filterStepKind uint8 + +const ( + filterStepPrefix filterStepKind = iota + filterStepSkipN + filterStepSkipThrough + filterStepMinLength +) + +type filterStep struct { + kind filterStepKind + str string + n int +} + +func planLike(like string) []filterStep { + tokens := tokenizeLike(like) + + steps := make([]filterStep, 0, len(tokens)) + var merged bool + for i, t := range tokens { + if merged { + merged = false + continue + } + + var step filterStep + hasPercent := strings.ContainsRune(t, '%') + underscores := strings.Count(t, "_") + switch { + case hasPercent && i+1 < len(tokens): + step = filterStep{ + kind: filterStepSkipThrough, + str: tokens[i+1], + n: underscores, + } + merged = true + case hasPercent: + step = filterStep{ + kind: filterStepMinLength, + n: underscores, + } + case underscores > 0: + step = filterStep{ + kind: filterStepSkipN, + n: underscores, + } + default: + step = filterStep{ + kind: filterStepPrefix, + str: t, + } + } + steps = append(steps, step) + } + + return steps +} + +func matchLike(key string, like ...filterStep) bool { + for i, step := range like { + switch step.kind { + case filterStepPrefix: + if !strings.HasPrefix(key, step.str) { + return false + } + key = key[len(step.str):] + case filterStepSkipN: + n := step.n + for j := 0; j < n; j++ { + _, len := utf8.DecodeRuneInString(key) + if len == 0 { + return false + } + key = key[len:] + } + case filterStepSkipThrough: + var skipped int + for skipped < step.n { + j := strings.Index(key, step.str) + switch j { + case -1: + return false + case 0: + _, len := utf8.DecodeRuneInString(key) + if len == 0 { + return false + } + key = key[len:] + skipped += len + default: + k := -1 + for k = range key[:j] { + } + skipped += k + 1 + + key = key[j:] + } + } + + remaining := like[i+1:] + for { + j := strings.Index(key, step.str) + switch { + case j == -1: + return false + case j > 0: + key = key[j:] + } + + if matchLike(key[len(step.str):], remaining...) { + return true + } + key = key[1:] + } + case filterStepMinLength: + if len(key) < step.n { + return false + } + + j := -1 + for j = range key { + } + return j+1 >= step.n + default: + panic("invalid step") + } + } + return key == "" +} diff --git a/like_test.go b/like_test.go new file mode 100644 index 000000000..9ff6be91c --- /dev/null +++ b/like_test.go @@ -0,0 +1,184 @@ +package pilosa + +import ( + "reflect" + "testing" +) + +func TestPlanLike(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + like string + plan []filterStep + match, nonmatch []string + }{ + { + name: "Empty", + like: "", + plan: []filterStep{}, + match: []string{""}, + nonmatch: []string{"a", " "}, + }, + { + name: "Exact", + like: "x", + plan: []filterStep{ + { + kind: filterStepPrefix, + str: "x", + }, + }, + match: []string{"x"}, + nonmatch: []string{"", "y", "z", "xy", "yx"}, + }, + { + name: "Anything", + like: "%", + plan: []filterStep{ + { + kind: filterStepMinLength, + n: 0, + }, + }, + match: []string{"", "a", "b", "ab"}, + }, + { + name: "Prefix", + like: "x%", + plan: []filterStep{ + { + kind: filterStepPrefix, + str: "x", + }, + { + kind: filterStepMinLength, + n: 0, + }, + }, + match: []string{"xy", "xyz", "xyzzy"}, + nonmatch: []string{"plugh", "yx", ""}, + }, + { + name: "Suffix", + like: "%x", + plan: []filterStep{ + { + kind: filterStepSkipThrough, + str: "x", + }, + }, + match: []string{"x", "xx", "ax"}, + nonmatch: []string{"", "a", "x^"}, + }, + { + name: "Sandwich", + like: "x%y", + plan: []filterStep{ + { + kind: filterStepPrefix, + str: "x", + }, + { + kind: filterStepSkipThrough, + str: "y", + }, + }, + match: []string{"xy", "xzy", "xyzzy"}, + nonmatch: []string{"plugh", ".xy.", ".x.y", "x.y."}, + }, + { + name: "SingleRune", + like: "_", + plan: []filterStep{ + { + kind: filterStepSkipN, + n: 1, + }, + }, + match: []string{"a", "á", "☺"}, + nonmatch: []string{"ab", "á", "h̷"}, + }, + { + name: "DoubleRune", + like: "__", + plan: []filterStep{ + { + kind: filterStepSkipN, + n: 2, + }, + }, + match: []string{"ab", "á", "h̷"}, + nonmatch: []string{"a", "á", "☺", "abc"}, + }, + { + name: "MiddleBlank", + like: "x_y", + plan: []filterStep{ + { + kind: filterStepPrefix, + str: "x", + }, + { + kind: filterStepSkipN, + n: 1, + }, + { + kind: filterStepPrefix, + str: "y", + }, + }, + match: []string{"x.y", "xay", "x y", "x⊕y"}, + nonmatch: []string{"x++y", "", "a"}, + }, + { + name: "MinLength", + like: "_%_", + plan: []filterStep{ + { + kind: filterStepMinLength, + n: 2, + }, + }, + match: []string{"ab", "á", "abc", "pilosa"}, + nonmatch: []string{"h", "á", ".", "☺"}, + }, + } + t.Run("Plan", func(t *testing.T) { + t.Parallel() + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + t.Parallel() + + plan := planLike(c.like) + if !reflect.DeepEqual(plan, c.plan) { + t.Errorf("incorrect plan: %v", plan) + } + }) + } + }) + t.Run("Match", func(t *testing.T) { + t.Parallel() + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + t.Parallel() + + for _, m := range c.match { + if !matchLike(m, c.plan...) { + t.Errorf("key %q was not matched", m) + } + } + for _, nm := range c.nonmatch { + if matchLike(nm, c.plan...) { + t.Errorf("key %q was matched", nm) + } + } + }) + } + }) +} From ed8e5a933e14c58ad24e6b1b75271a27e5882146 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Tue, 7 Jul 2020 12:43:50 -0400 Subject: [PATCH 2/7] optimize & document & test like matcher optimize suffix matching add some descriptive comments to the like matcher test all paths in the like tokenizer and matcher --- like.go | 85 ++++++++++++++++++++++++++++++++++++++++++++-------- like_test.go | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 125 insertions(+), 16 deletions(-) diff --git a/like.go b/like.go index db92232d7..31eb0ab1a 100644 --- a/like.go +++ b/like.go @@ -5,21 +5,29 @@ import ( "unicode/utf8" ) +// tokenizeLike turns a "like" pattern into a list of tokens. +// Every token is either a string to exactly match or a combination of % and _ placeholders. func tokenizeLike(like string) []string { var tokens []string for like != "" { var token string i := strings.IndexAny(like, "%_") - switch { - case i == 0: + switch i { + case 0: + // Generate a token of placeholders. + + // Iterate bytewise over the string to find the end of the token. + // The % and _ characters are ASCII, so we do not have to worry about Unicode right here. j := 1 for j < len(like) && (like[j] == '%' || like[j] == '_') { j++ } token, like = like[:j], like[j:] - case i < 0: + case -1: + // There are no more placeholders - generate the last token. token, like = like, "" default: + // Generate an exact match token. token, like = like[:i], like[i:] } tokens = append(tokens, token) @@ -27,37 +35,50 @@ func tokenizeLike(like string) []string { return tokens } +// filterStepKind is a kind of step in a like filter. type filterStepKind uint8 const ( - filterStepPrefix filterStepKind = iota - filterStepSkipN - filterStepSkipThrough - filterStepMinLength + filterStepPrefix filterStepKind = iota // x... + filterStepSkipN // __... + filterStepSkipThrough // %x... + filterStepSuffix // %x + filterStepMinLength // _% ) +// filterStep is a step in a like filter. type filterStep struct { + // kind is the step kind. kind filterStepKind - str string - n int + + // str is the substring for a prefix/skipthrough/suffix step. + str string + + // n is the number of underscores in the step (if relevant). + n int } +// planLike generates a filtering plan for a like pattern. func planLike(like string) []filterStep { + // Tokenize the like pattern. tokens := tokenizeLike(like) steps := make([]filterStep, 0, len(tokens)) var merged bool for i, t := range tokens { if merged { + // The token was already merged into the previous step. merged = false continue } + // Convert the token to a step. var step filterStep hasPercent := strings.ContainsRune(t, '%') underscores := strings.Count(t, "_") switch { case hasPercent && i+1 < len(tokens): + // Generate a step to skip through the next token. step = filterStep{ kind: filterStepSkipThrough, str: tokens[i+1], @@ -65,16 +86,19 @@ func planLike(like string) []filterStep { } merged = true case hasPercent: + // Generate a terminating step to absorb the remainder of the string. step = filterStep{ kind: filterStepMinLength, n: underscores, } case underscores > 0: + // Generate a step to absorb _ placeholders. step = filterStep{ kind: filterStepSkipN, n: underscores, } default: + // Generate a step to process an exact match of the beginning of a string. step = filterStep{ kind: filterStepPrefix, str: t, @@ -83,18 +107,26 @@ func planLike(like string) []filterStep { steps = append(steps, step) } + // Optimize suffix matching. + if len(steps) > 0 && steps[len(steps)-1].kind == filterStepSkipThrough { + steps[len(steps)-1].kind = filterStepSuffix + } + return steps } +// matchLike matches a string using a like plan. func matchLike(key string, like ...filterStep) bool { for i, step := range like { switch step.kind { case filterStepPrefix: + // Match a prefix. if !strings.HasPrefix(key, step.str) { return false } key = key[len(step.str):] case filterStepSkipN: + // Skip some placeholders. n := step.n for j := 0; j < n; j++ { _, len := utf8.DecodeRuneInString(key) @@ -104,20 +136,25 @@ func matchLike(key string, like ...filterStep) bool { key = key[len:] } case filterStepSkipThrough: + // Skip through a string. + + // Skip through placeholders. var skipped int for skipped < step.n { j := strings.Index(key, step.str) switch j { case -1: + // There are no more matches. return false case 0: + // Skip a single rune to ensure forward progress. + // This is somewhat inefficient since we have to search the string again next time. + // This will hopefully not have to be used very frequently. _, len := utf8.DecodeRuneInString(key) - if len == 0 { - return false - } key = key[len:] skipped += len default: + // Skip until the substring and count the skipped runes. k := -1 for k = range key[:j] { } @@ -127,33 +164,55 @@ func matchLike(key string, like ...filterStep) bool { } } + // Iterate through the substring matches until the rest of the pattern matches. remaining := like[i+1:] for { + // Find the next substring match. j := strings.Index(key, step.str) switch { case j == -1: + // There are no more matches. return false case j > 0: + // Skip the data before the substring. key = key[j:] } + // Apply the rest of the filter. if matchLike(key[len(step.str):], remaining...) { + // This instance matches, no need to search any more. return true } - key = key[1:] + + // Skip the first rune of the substring so we do not rescan this substring match. + _, len := utf8.DecodeRuneInString(key) + key = key[len:] } + case filterStepSuffix: + // Match a suffix. + j := strings.LastIndex(key, step.str) + if j < step.n { + return false + } + key = key[j+len(step.str):] case filterStepMinLength: if len(key) < step.n { + // The string is definitely too short. return false } + // Count the runes. j := -1 for j = range key { } + + // Check if the string is long enough. return j+1 >= step.n default: panic("invalid step") } } + + // If there is any unmatched data left, this is not a match. return key == "" } diff --git a/like_test.go b/like_test.go index 9ff6be91c..b6aee7c78 100644 --- a/like_test.go +++ b/like_test.go @@ -65,7 +65,7 @@ func TestPlanLike(t *testing.T) { like: "%x", plan: []filterStep{ { - kind: filterStepSkipThrough, + kind: filterStepSuffix, str: "x", }, }, @@ -81,13 +81,63 @@ func TestPlanLike(t *testing.T) { str: "x", }, { - kind: filterStepSkipThrough, + kind: filterStepSuffix, str: "y", }, }, match: []string{"xy", "xzy", "xyzzy"}, nonmatch: []string{"plugh", ".xy.", ".x.y", "x.y."}, }, + { + name: "DoubleDeckerSandwich", + like: "x%y%z", + plan: []filterStep{ + { + kind: filterStepPrefix, + str: "x", + }, + { + kind: filterStepSkipThrough, + str: "y", + }, + { + kind: filterStepSuffix, + str: "z", + }, + }, + match: []string{"xyz", "xzyzz", "x.y.z", "x.y.y..z"}, + nonmatch: []string{"plugh", ".xyz.", ".x.y.z", "x.y.z."}, + }, + { + name: "Skips", + like: "a_b_%_c_%_%_d", + plan: []filterStep{ + { + kind: filterStepPrefix, + str: "a", + }, + { + kind: filterStepSkipN, + n: 1, + }, + { + kind: filterStepPrefix, + str: "b", + }, + { + kind: filterStepSkipThrough, + str: "c", + n: 2, + }, + { + kind: filterStepSuffix, + str: "d", + n: 3, + }, + }, + match: []string{"a1b234c5678d"}, + nonmatch: []string{"abcd", "a1b2345678d", "a1b2c5678d"}, + }, { name: "SingleRune", like: "_", @@ -98,7 +148,7 @@ func TestPlanLike(t *testing.T) { }, }, match: []string{"a", "á", "☺"}, - nonmatch: []string{"ab", "á", "h̷"}, + nonmatch: []string{"ab", "á", "h̷", ""}, }, { name: "DoubleRune", From 0611ef3418643eabebfca3264b266dbc59953be9 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Tue, 7 Jul 2020 15:32:34 -0400 Subject: [PATCH 3/7] initial implementation of Rows like --- executor.go | 13 +++++++++++++ fragment.go | 16 ++++++++++++++++ pql/ast.go | 14 ++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/executor.go b/executor.go index dcfaf39ed..2c736badd 100644 --- a/executor.go +++ b/executor.go @@ -2431,6 +2431,14 @@ func (e *executor) executeRowsShard(ctx context.Context, tx Tx, index string, fi limit = int(lim) } + var likeErr chan error + if like, hasLike, err := c.StringArg("like"); err != nil { + return nil, errors.Wrap(err, "getting like pattern") + } else if hasLike { + likeErr = make(chan error, 1) + filters = append(filters, filterLike(like, f.TranslateStore(), likeErr)) + } + for _, view := range views { if err := ctx.Err(); err != nil { return nil, err @@ -2444,6 +2452,11 @@ func (e *executor) executeRowsShard(ctx context.Context, tx Tx, index string, fi if err != nil { return nil, err } + select { + case err = <-likeErr: + return nil, err + default: + } rowIDs = rowIDs.merge(viewRows, limit) } diff --git a/fragment.go b/fragment.go index b31e8d8ff..d052a3a3c 100644 --- a/fragment.go +++ b/fragment.go @@ -2802,6 +2802,22 @@ func filterColumn(col uint64) rowFilter { } } +func filterLike(like string, t TranslateStore, e chan error) rowFilter { + plan := planLike(like) + + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + keyStr, err := t.TranslateID(rowID) + if err != nil { + select { + case e <- err: + default: + } + return false, true + } + return matchLike(keyStr, plan...), false + } +} + // TODO: this works, but it would be more performant if the fragment could seek // to the next row in the rows list rather than asking the filter for each // container serially. The container iterator would need to expose a seek diff --git a/pql/ast.go b/pql/ast.go index e93a2aaf5..4195e66bf 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -384,6 +384,7 @@ var callInfoByFunc = map[string]callInfo{ "previous": nil, "from": nil, "to": nil, + "like": "", }, }, "Shift": {allowUnknown: false, @@ -662,6 +663,19 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } +func (c *Call) StringArg(key string) (string, bool, error) { + val, ok := c.Args[key] + if !ok { + return "", false, nil + } + switch tval := val.(type) { + case string: + return tval, true, nil + default: + return "", true, fmt.Errorf("unexpected type %T in StringArg, val %v", tval, tval) + } +} + // CallArg is for reading the value at key from call.Args as a Call. If the // key is not in Call.Args, the value of the returned value will be nil, and // the error will be nil. An error is returned if the value is not a Call. From 227881cb67a87b3de1a8cf25d5dbbf42394e872c Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Tue, 7 Jul 2020 15:41:29 -0400 Subject: [PATCH 4/7] simplify suffix matching --- like.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/like.go b/like.go index 31eb0ab1a..cb66a9d91 100644 --- a/like.go +++ b/like.go @@ -190,11 +190,18 @@ func matchLike(key string, like ...filterStep) bool { } case filterStepSuffix: // Match a suffix. - j := strings.LastIndex(key, step.str) - if j < step.n { + if !strings.HasSuffix(key, step.str) { + // Suffix not present. return false } - key = key[j+len(step.str):] + if step.n <= 0 { + // No skip length check necessary. + return true + } + + // Check length of the substring before the suffix. + key = key[:len(key)-len(step.str)] + fallthrough case filterStepMinLength: if len(key) < step.n { // The string is definitely too short. From 7a16ffff68e095c142597ad5647e280a8944594a Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 8 Jul 2020 12:06:08 -0400 Subject: [PATCH 5/7] add UnionRows query --- executor.go | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++ pql/ast.go | 5 ++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 2c736badd..42abe099c 100644 --- a/executor.go +++ b/executor.go @@ -495,6 +495,111 @@ func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Quer return results, nil } +// preprocessQuery expands any calls that need preprocessing. +// So far, this only needs to process UnionRows. +func (e *executor) preprocessQuery(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*pql.Call, error) { + switch c.Name { + case "UnionRows": + // Turn UnionRows(Rows(...)) into Union(Row(...), ...). + var rows []*pql.Call + for _, child := range c.Children { + // Check that we can use the call. + switch child.Name { + case "Rows": + case "TopN": + default: + return nil, errors.Errorf("cannot use %v as a rows query", child) + } + + // Execute the call. + rowsResult, err := e.executeCall(ctx, tx, index, child, shards, opt) + if err != nil { + return nil, err + } + + // Turn the results into rows calls. + var resultRows []*pql.Call + switch rowsResult := rowsResult.(type) { + case *PairsField: + // Translate pairs into rows calls. + for _, p := range rowsResult.Pairs { + var val interface{} + switch { + case p.Key != "": + val = p.Key + default: + val = p.ID + } + resultRows = append(resultRows, &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + rowsResult.Field: val, + }, + }) + } + case RowIDs: + // Translate Row IDs into Row calls. + for _, id := range rowsResult { + resultRows = append(resultRows, &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + child.Args["_field"].(string): id, + }, + }) + } + default: + return nil, errors.Errorf("unexpected Rows type %T", rowsResult) + } + + // Propogate any special properties of the call. + switch child.Name { + case "Rows": + // Propogate "from" time, if set. + if v, ok := child.Args["from"]; ok { + for _, rowCall := range resultRows { + rowCall.Args["from"] = v + } + } + + // Propogate "to" time, if set. + if v, ok := child.Args["to"]; ok { + for _, rowCall := range resultRows { + rowCall.Args["to"] = v + } + } + } + + rows = append(rows, resultRows...) + } + + // Generate a Union call over the rows. + return &pql.Call{ + Name: "Union", + Children: rows, + }, nil + + default: + // Recurse through child calls. + out := make([]*pql.Call, len(c.Children)) + var changed bool + for i, child := range c.Children { + res, err := e.preprocessQuery(ctx, tx, index, child, shards, opt) + if err != nil { + return nil, err + } + if res != child { + changed = true + } + out[i] = res + } + if changed { + c = c.Clone() + c.Children = out + } + return c, nil + } +} + // executeCall executes a call. func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") @@ -534,6 +639,12 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql. } } + // Preprocess the query. + c, err := e.preprocessQuery(ctx, tx, index, c, shards, opt) + if err != nil { + return nil, err + } + // Special handling for mutation and top-n calls. if op, ok := e.additionalCountOps[c.Name]; ok { statFn() diff --git a/pql/ast.go b/pql/ast.go index 4195e66bf..9d65bb3c5 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -392,8 +392,9 @@ var callInfoByFunc = map[string]callInfo{ "n": int64(0), }, }, - "Union": {allowUnknown: false}, - "Xor": {allowUnknown: false}, + "Union": {allowUnknown: false}, + "UnionRows": {allowUnknown: false}, + "Xor": {allowUnknown: false}, // things that take _field "TopN": allowUnderField, From c9edccb267855e19eb480afcc4f3c6049d310f6f Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 8 Jul 2020 12:50:36 -0400 Subject: [PATCH 6/7] add executor tests and license headers for like & UnionRows --- executor_test.go | 29 +++++++++++++++++++++++++++++ like.go | 14 ++++++++++++++ like_test.go | 14 ++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/executor_test.go b/executor_test.go index d1ae63cb4..4a39a7713 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4742,6 +4742,10 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { q: `Rows(f, previous="1", limit=0, column="0")`, exp: []string{}, }, + { + q: `Rows(f, like="__")`, + exp: []string{"10", "11", "12", "13", "14", "15", "16", "17", "18"}, + }, } for i, test := range tests { @@ -5843,6 +5847,31 @@ func TestExecutor_Execute_TopNDistinct(t *testing.T) { }) } +func Test_Executor_Execute_UnionRows(t *testing.T) { + c := test.MustRunCluster(t, 2) + defer c.Close() + + c.CreateField(t, "i", pilosa.IndexOptions{}, "s", + pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 50000), + ) + + // Populate data. + c.Query(t, "i", ` + Set(0, s=1) + Set(1, s=2) + Set(2, s=3) + Set(3, s=1) + Set(3, s=5) + `) + + if res := c.Query(t, "i", `Count(UnionRows(TopN(s, n=1)))`); res.Results[0] != uint64(2) { + t.Errorf("expected 2 columns, got %v", res.Results[0]) + } + if res := c.Query(t, "i", `Count(UnionRows(Rows(s)))`); res.Results[0] != uint64(4) { + t.Errorf("expected 4 columns, got %v", res.Results[0]) + } +} + func TestTimelessClearRegression(t *testing.T) { data, err := ioutil.ReadFile("testdata/timeRegressionSchema.json") if err != nil { diff --git a/like.go b/like.go index cb66a9d91..d3fbebe54 100644 --- a/like.go +++ b/like.go @@ -1,3 +1,17 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( diff --git a/like_test.go b/like_test.go index b6aee7c78..1382901ec 100644 --- a/like_test.go +++ b/like_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa import ( From 4bacab6524f726aa00e9a2062f4b5730415029b5 Mon Sep 17 00:00:00 2001 From: Jaden Weiss Date: Wed, 8 Jul 2020 13:08:11 -0400 Subject: [PATCH 7/7] update query language docs to include like and UnionRows --- docs/query-language.md | 45 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index fb7e510ee..310a4116c 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -52,6 +52,7 @@ curl localhost:10101/index/repository/query \ * `CALL` Any query. * `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Not`. * `ROWS_CALL` A query that returns a `Rows` result (i.e. a list of row IDs). Currently only the `Rows` query. +* `ROWSET_CALL` A query that returns a set of rows. Currently only the `Rows` and `TopN` queries. * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`). ### Write Operations @@ -796,7 +797,7 @@ Options(Row(f1=10), shards=[0, 2]) **Spec:** ``` -Rows(, previous=, limit=, column=, from=, to=) +Rows(, previous=, limit=, column=, from=, to=, like=) ``` **Description:** @@ -817,6 +818,10 @@ If the field is of type `time`, the `from` and `to` arguments can be provided to restrict the result to a specific time span. If `from` and `to` are not provided, the full range of existing data will be queried. +If `like` is given, only keys matching a pattern will be selected. +A `like` pattern may use `_` as a placeholder to match a single UTF-8 codepoint, and `%` to match 0 or more codepoints. +All other characters will be matched exactly. + **Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.` **Examples:** @@ -834,7 +839,15 @@ With keys: Rows(job) ``` ```response -{"rows":null,"keys":["engineer","management","student""]} +{"rows":null,"keys":["engineer","management","student"]} +``` + +With `like`: +```request +Rows(job, like="%t") +``` +```response +{"rows":null,"keys":["management","student"]} ``` #### Group By @@ -923,3 +936,31 @@ GroupBy(Rows(age), Rows(job), limit=7, filter=Row(country=USA)) {"group":[{"field":"age","rowID":22},{"field":"job","rowKey":"student"}],"count":3}, {"group":[{"field":"age","rowID":29},{"field":"job","rowKey":"management"}],"count":7}] ``` + +#### UnionRows + +**Spec:** + +``` +UnionRows([ROWSET_CALL ...]) +``` + +**Description:** + +UnionRows performs a logical OR on the rows matched by the results of all `ROWSET_CALL` queries passed to it. + +**Result Type:** object with attrs and bits + +attrs will always be empty + +**Examples:** + +Query columns with a bit set in any row (repositories that are starred by any user): +```request +UnionRows(Rows(stargazer)) +``` +```response +{"attrs":{},"columns":[10, 20, 30]} +``` + +* columns are repositories that were starred by any user \ No newline at end of file