From f9a265f94dbf0454ab21cab7b81521510fff7417 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Jul 2018 17:23:54 -0500 Subject: [PATCH 01/17] fix places where empty IndexOptions were being used --- holder.go | 3 +-- server.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/holder.go b/holder.go index 6292d9c59..c2e8f5a6e 100644 --- a/holder.go +++ b/holder.go @@ -248,8 +248,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { func (h *Holder) applySchema(schema *Schema) error { // Create indexes that don't exist. for _, index := range schema.Indexes { - opt := IndexOptions{} - idx, err := h.CreateIndexIfNotExists(index.Name, opt) + idx, err := h.CreateIndexIfNotExists(index.Name, index.Options) if err != nil { return errors.Wrap(err, "creating index") } diff --git a/server.go b/server.go index badf06f8c..2cc4accdc 100644 --- a/server.go +++ b/server.go @@ -457,8 +457,8 @@ func (s *Server) receiveMessage(m Message) error { } idx.setRemoteMaxShard(obj.Shard) case *CreateIndexMessage: - opt := IndexOptions{} - _, err := s.holder.CreateIndex(obj.Index, opt) + opt := obj.Meta + _, err := s.holder.CreateIndex(obj.Index, *opt) if err != nil { return err } From a427836a9abb1e5d9a117046d0bdf44a6473af20 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 24 Jul 2018 10:06:20 -0500 Subject: [PATCH 02/17] Fix errant references of server.primaryTranslateStore to server.translateFile --- api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index afb0e2379..b29b3c1fc 100644 --- a/api.go +++ b/api.go @@ -135,9 +135,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er } // Translate column attributes, if necessary. - if api.server.primaryTranslateStore != nil { + if api.server.translateFile != nil { for _, col := range resp.ColumnAttrSets { - v, err := api.server.primaryTranslateStore.TranslateColumnToString(req.Index, col.ID) + v, err := api.server.translateFile.TranslateColumnToString(req.Index, col.ID) if err != nil { return resp, err } @@ -760,7 +760,7 @@ func (api *API) ResizeAbort() error { const translateStoreBufferSize = 65536 func (api *API) GetTranslateData(ctx context.Context, w io.WriteCloser, offset int64) error { - rc, err := api.server.primaryTranslateStore.Reader(ctx, offset) + rc, err := api.server.translateFile.Reader(ctx, offset) if err != nil { return errors.Wrap(err, "read from translate store") } From ca800c683e87191ef158a3d10ab1f51a818f95cd Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 24 Jul 2018 14:57:06 -0500 Subject: [PATCH 03/17] Add test for cluster translator --- server/handler_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/server/handler_test.go b/server/handler_test.go index bac09a1cc..2a319c41b 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -655,6 +655,34 @@ func TestHandler_Endpoints(t *testing.T) { }) } +func TestClusterTranslator(t *testing.T) { + cluster := make(test.Cluster, 2) + cluster[0] = test.NewCommandNode(true) + cluster[0].Config.Gossip.Port = "0" + cluster[0].Start() + httpTranslateStore := http.NewTranslateStore(cluster[0].URL()) + cluster[1] = test.NewCommandNode(false, + server.OptCommandServerOptions( + pilosa.OptServerPrimaryTranslateStore(httpTranslateStore), + ), + ) + cluster[1].Config.Gossip.Port = "0" + cluster[1].Config.Gossip.Seeds = []string{cluster[0].GossipAddress()} + cluster[1].Start() + + test.MustDo("POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}") + test.MustDo("POST", cluster[0].URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}") + + test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")") + + result0 := test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body + result1 := test.MustDo("POST", cluster[1].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body + + if result0 != result1 { + t.Fatalf("`%s` != `%s`", result0, result1) + } +} + func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { dec := json.NewDecoder(r) err := dec.Decode(&ret) From f90a6dff8bdb48edb3cc274ee02925ce873da792 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 26 Jul 2018 08:11:30 -0500 Subject: [PATCH 04/17] fix topn spec and example in docs --- docs/query-language.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 4141ea38f..a61a9bf09 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -419,7 +419,7 @@ Count(Row(stargazer=1)) **Spec:** ``` -TopN([ROW_CALL], , [n=UINT], +TopN(, [ROW_CALL], [n=UINT], [attrName=, attrValues=<[]ATTR_VALUE>]) ``` @@ -468,7 +468,7 @@ TopN(stargazer, n=2) Filter based on an existing row: ```request -TopN(Row(language=1), stargazer, n=2) +TopN(stargazer, Row(language=1), n=2) ``` ```response {"results":[[{"id":1240,"count":35},{"id":7508,"count":32}]]} From dfd529b2f5d2b38b0c9dc2fab28521114a0c726d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 27 Jul 2018 10:50:04 -0500 Subject: [PATCH 05/17] update parser to handle row keys on SetRowAttrs() --- executor.go | 6 +- executor_test.go | 70 +- index.go | 12 +- pql/pql.peg | 6 +- pql/pql.peg.go | 2040 +++++++++++++++++++++++--------------------- pql/pqlpeg_test.go | 58 ++ test/index.go | 4 +- 7 files changed, 1181 insertions(+), 1015 deletions(-) diff --git a/executor.go b/executor.go index 52f851e5a..6f0e0ce8b 100644 --- a/executor.go +++ b/executor.go @@ -1256,7 +1256,7 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal rowID, ok, err := c.UintArg("_" + rowLabel) if err != nil { - return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) + return nil, errors.Wrap(err, "reading SetRowAttrs() row") } else if !ok { return nil, fmt.Errorf("SetRowAttrs row field '%v' required", rowLabel) } @@ -1550,6 +1550,10 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { colKey = "_" + columnLabel fieldName, _ = c.FieldArg() rowKey = fieldName + } else if c.Name == "SetRowAttrs" { + // Positional args in new PQL syntax require special handling here. + rowKey = "_" + rowLabel + fieldName = callArgString(c, "_field") } else { colKey = "col" fieldName = callArgString(c, "field") diff --git a/executor_test.go b/executor_test.go index 034e5bc70..05c99bfd1 100644 --- a/executor_test.go +++ b/executor_test.go @@ -471,29 +471,59 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { t.Fatal(err) } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } - - // Set two attrs on f/10. - // Also set attrs on other bitmaps and fields to test isolation. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { - t.Fatal(err) - } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil { - t.Fatal(err) - } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil { - t.Fatal(err) - } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("kf", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) } - f := hldr.Field("i", "f") - if m, err := f.RowAttrStore().Attrs(10); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { - t.Fatalf("unexpected bitmap attr: %#v", m) - } + t.Run("rowID", func(t *testing.T) { + // Set two attrs on f/10. + // Also set attrs on other bitmaps and fields to test isolation. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil { + t.Fatal(err) + } + + f := hldr.Field("i", "f") + if m, err := f.RowAttrStore().Attrs(10); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { + t.Fatalf("unexpected bitmap attr: %#v", m) + } + }) + + t.Run("rowKey", func(t *testing.T) { + // Set two attrs on f/10. + // Also set attrs on other bitmaps and fields to test isolation. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", foo="bar")`}); err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row200", YYY=1)`}); err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", baz=123, bat=true)`}); err != nil { + t.Fatal(err) + } + + result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}) + if err != nil { + t.Fatal(err) + } + spew.Dump(result) + + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}); err != nil { + t.Fatal(err) + } else if attrs := result.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { + t.Fatalf("unexpected attrs: %+v", attrs) + } + }) } // Ensure a TopN() query can be executed. diff --git a/index.go b/index.go index 96d45f85c..d9338cceb 100644 --- a/index.go +++ b/index.go @@ -292,7 +292,7 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { } // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. -func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, error) { +func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() @@ -301,11 +301,13 @@ func (i *Index) CreateFieldIfNotExists(name string, opts FieldOption) (*Field, e return f, nil } - // Apply functional option. + // Apply functional options. fo := FieldOptions{} - err := opts(&fo) - if err != nil { - return nil, errors.Wrap(err, "applying option") + for _, opt := range opts { + err := opt(&fo) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } } return i.createField(name, fo) diff --git a/pql/pql.peg b/pql/pql.peg index f55267e1f..33896f8ad 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -7,7 +7,7 @@ type PQL Peg { Calls <- sp (Call sp)* !. Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close {p.endCall()} - / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma uintrow comma args close {p.endCall()} + / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()} / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} @@ -60,6 +60,10 @@ col <- ( {p.addPosNum("_col", buffer[begin:end])} / '\'' '\'' {p.addPosStr("_col", buffer[begin:end])} / '"' '"' {p.addPosStr("_col", buffer[begin:end])} ) +row <- ( {p.addPosNum("_row", buffer[begin:end])} + / '\'' '\'' {p.addPosStr("_row", buffer[begin:end])} + / '"' '"' {p.addPosStr("_row", buffer[begin:end])} + ) open <- '(' sp close <- ')' sp diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 7092b8f1f..21166d0ca 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -39,6 +39,7 @@ const ( ruleuint ruleuintrow rulecol + rulerow ruleopen ruleclose rulesp @@ -95,6 +96,9 @@ const ( ruleAction42 ruleAction43 ruleAction44 + ruleAction45 + ruleAction46 + ruleAction47 ) var rul3s = [...]string{ @@ -122,6 +126,7 @@ var rul3s = [...]string{ "uint", "uintrow", "col", + "row", "open", "close", "sp", @@ -178,6 +183,9 @@ var rul3s = [...]string{ "Action42", "Action43", "Action44", + "Action45", + "Action46", + "Action47", } type token32 struct { @@ -294,7 +302,7 @@ type PQL struct { Buffer string buffer []rune - rules [80]func() bool + rules [84]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -475,6 +483,12 @@ func (p *PQL) Execute() { case ruleAction43: p.addPosStr("_col", buffer[begin:end]) case ruleAction44: + p.addPosNum("_row", buffer[begin:end]) + case ruleAction45: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction46: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction47: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -587,7 +601,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma uintrow comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -636,7 +650,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction44, position) + add(ruleAction47, position) } add(ruletimestamp, position12) } @@ -713,16 +727,62 @@ func (p *PQL) Init() { { position18 := position { - position19 := position - if !_rules[ruleuint]() { + position19, tokenIndex19 := position, tokenIndex + { + position21 := position + if !_rules[ruleuint]() { + goto l20 + } + add(rulePegText, position21) + } + { + add(ruleAction44, position) + } + goto l19 + l20: + position, tokenIndex = position19, tokenIndex19 + if buffer[position] != rune('\'') { + goto l23 + } + position++ + { + position24 := position + if !_rules[rulesinglequotedstring]() { + goto l23 + } + add(rulePegText, position24) + } + if buffer[position] != rune('\'') { + goto l23 + } + position++ + { + add(ruleAction45, position) + } + goto l19 + l23: + position, tokenIndex = position19, tokenIndex19 + if buffer[position] != rune('"') { goto l16 } - add(rulePegText, position19) + position++ + { + position26 := position + if !_rules[ruledoublequotedstring]() { + goto l16 + } + add(rulePegText, position26) + } + if buffer[position] != rune('"') { + goto l16 + } + position++ + { + add(ruleAction46, position) + } } - { - add(ruleAction40, position) - } - add(ruleuintrow, position18) + l19: + add(rulerow, position18) } if !_rules[rulecomma]() { goto l16 @@ -740,360 +800,360 @@ func (p *PQL) Init() { l16: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('S') { - goto l22 + goto l29 } position++ if buffer[position] != rune('e') { - goto l22 + goto l29 } position++ if buffer[position] != rune('t') { - goto l22 + goto l29 } position++ if buffer[position] != rune('C') { - goto l22 + goto l29 } position++ if buffer[position] != rune('o') { - goto l22 + goto l29 } position++ if buffer[position] != rune('l') { - goto l22 + goto l29 } position++ if buffer[position] != rune('u') { - goto l22 + goto l29 } position++ if buffer[position] != rune('m') { - goto l22 + goto l29 } position++ if buffer[position] != rune('n') { - goto l22 + goto l29 } position++ if buffer[position] != rune('A') { - goto l22 + goto l29 } position++ if buffer[position] != rune('t') { - goto l22 + goto l29 } position++ if buffer[position] != rune('t') { - goto l22 + goto l29 } position++ if buffer[position] != rune('r') { - goto l22 + goto l29 } position++ if buffer[position] != rune('s') { - goto l22 + goto l29 } position++ { add(ruleAction4, position) } if !_rules[ruleopen]() { - goto l22 + goto l29 } if !_rules[rulecol]() { - goto l22 + goto l29 } if !_rules[rulecomma]() { - goto l22 + goto l29 } if !_rules[ruleargs]() { - goto l22 + goto l29 } if !_rules[ruleclose]() { - goto l22 + goto l29 } { add(ruleAction5, position) } goto l7 - l22: + l29: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('C') { - goto l25 + goto l32 } position++ if buffer[position] != rune('l') { - goto l25 + goto l32 } position++ if buffer[position] != rune('e') { - goto l25 + goto l32 } position++ if buffer[position] != rune('a') { - goto l25 + goto l32 } position++ if buffer[position] != rune('r') { - goto l25 + goto l32 } position++ { add(ruleAction6, position) } if !_rules[ruleopen]() { - goto l25 + goto l32 } if !_rules[rulecol]() { - goto l25 + goto l32 } if !_rules[rulecomma]() { - goto l25 + goto l32 } if !_rules[ruleargs]() { - goto l25 + goto l32 } if !_rules[ruleclose]() { - goto l25 + goto l32 } { add(ruleAction7, position) } goto l7 - l25: + l32: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('T') { - goto l28 + goto l35 } position++ if buffer[position] != rune('o') { - goto l28 + goto l35 } position++ if buffer[position] != rune('p') { - goto l28 + goto l35 } position++ if buffer[position] != rune('N') { - goto l28 + goto l35 } position++ { add(ruleAction8, position) } if !_rules[ruleopen]() { - goto l28 + goto l35 } if !_rules[ruleposfield]() { - goto l28 + goto l35 } { - position30, tokenIndex30 := position, tokenIndex + position37, tokenIndex37 := position, tokenIndex if !_rules[rulecomma]() { - goto l30 + goto l37 } if !_rules[ruleallargs]() { - goto l30 + goto l37 } - goto l31 - l30: - position, tokenIndex = position30, tokenIndex30 + goto l38 + l37: + position, tokenIndex = position37, tokenIndex37 } - l31: + l38: if !_rules[ruleclose]() { - goto l28 + goto l35 } { add(ruleAction9, position) } goto l7 - l28: + l35: position, tokenIndex = position7, tokenIndex7 if buffer[position] != rune('R') { - goto l33 + goto l40 } position++ if buffer[position] != rune('a') { - goto l33 + goto l40 } position++ if buffer[position] != rune('n') { - goto l33 + goto l40 } position++ if buffer[position] != rune('g') { - goto l33 + goto l40 } position++ if buffer[position] != rune('e') { - goto l33 + goto l40 } position++ { add(ruleAction10, position) } if !_rules[ruleopen]() { - goto l33 + goto l40 } { - position35, tokenIndex35 := position, tokenIndex + position42, tokenIndex42 := position, tokenIndex { - position37 := position + position44 := position if !_rules[rulefield]() { - goto l36 + goto l43 } if !_rules[rulesp]() { - goto l36 + goto l43 } if buffer[position] != rune('=') { - goto l36 + goto l43 } position++ if !_rules[rulesp]() { - goto l36 + goto l43 } if !_rules[rulevalue]() { - goto l36 + goto l43 } if !_rules[rulecomma]() { - goto l36 + goto l43 } { - position38 := position + position45 := position if !_rules[ruletimestampfmt]() { - goto l36 + goto l43 } - add(rulePegText, position38) + add(rulePegText, position45) } { add(ruleAction26, position) } if !_rules[rulecomma]() { - goto l36 + goto l43 } { - position40 := position + position47 := position if !_rules[ruletimestampfmt]() { - goto l36 + goto l43 } - add(rulePegText, position40) + add(rulePegText, position47) } { add(ruleAction27, position) } - add(ruletimerange, position37) + add(ruletimerange, position44) } - goto l35 - l36: - position, tokenIndex = position35, tokenIndex35 + goto l42 + l43: + position, tokenIndex = position42, tokenIndex42 { - position43 := position + position50 := position { add(ruleAction21, position) } if !_rules[rulecondint]() { - goto l42 + goto l49 } if !_rules[rulecondLT]() { - goto l42 + goto l49 } { - position45 := position + position52 := position { - position46 := position + position53 := position if !_rules[rulefieldExpr]() { - goto l42 + goto l49 } - add(rulePegText, position46) + add(rulePegText, position53) } if !_rules[rulesp]() { - goto l42 + goto l49 } { add(ruleAction25, position) } - add(rulecondfield, position45) + add(rulecondfield, position52) } if !_rules[rulecondLT]() { - goto l42 + goto l49 } if !_rules[rulecondint]() { - goto l42 + goto l49 } { add(ruleAction22, position) } - add(ruleconditional, position43) + add(ruleconditional, position50) } - goto l35 - l42: - position, tokenIndex = position35, tokenIndex35 + goto l42 + l49: + position, tokenIndex = position42, tokenIndex42 if !_rules[rulearg]() { - goto l33 + goto l40 } } - l35: + l42: if !_rules[ruleclose]() { - goto l33 + goto l40 } { add(ruleAction11, position) } goto l7 - l33: + l40: position, tokenIndex = position7, tokenIndex7 { - position50 := position + position57 := position { - position51 := position + position58 := position { - position52, tokenIndex52 := position, tokenIndex + position59, tokenIndex59 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l53 + goto l60 } position++ - goto l52 - l53: - position, tokenIndex = position52, tokenIndex52 + goto l59 + l60: + position, tokenIndex = position59, tokenIndex59 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l52: - l54: + l59: + l61: { - position55, tokenIndex55 := position, tokenIndex + position62, tokenIndex62 := position, tokenIndex { - position56, tokenIndex56 := position, tokenIndex + position63, tokenIndex63 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l57 + goto l64 } position++ - goto l56 - l57: - position, tokenIndex = position56, tokenIndex56 + goto l63 + l64: + position, tokenIndex = position63, tokenIndex63 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l58 + goto l65 } position++ - goto l56 - l58: - position, tokenIndex = position56, tokenIndex56 + goto l63 + l65: + position, tokenIndex = position63, tokenIndex63 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l55 + goto l62 } position++ } - l56: - goto l54 - l55: - position, tokenIndex = position55, tokenIndex55 + l63: + goto l61 + l62: + position, tokenIndex = position62, tokenIndex62 } - add(ruleIDENT, position51) + add(ruleIDENT, position58) } - add(rulePegText, position50) + add(rulePegText, position57) } { add(ruleAction12, position) @@ -1105,15 +1165,15 @@ func (p *PQL) Init() { goto l5 } { - position60, tokenIndex60 := position, tokenIndex + position67, tokenIndex67 := position, tokenIndex if !_rules[rulecomma]() { - goto l60 + goto l67 } - goto l61 - l60: - position, tokenIndex = position60, tokenIndex60 + goto l68 + l67: + position, tokenIndex = position67, tokenIndex67 } - l61: + l68: if !_rules[ruleclose]() { goto l5 } @@ -1131,232 +1191,232 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position63, tokenIndex63 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex { - position64 := position + position71 := position { - position65, tokenIndex65 := position, tokenIndex + position72, tokenIndex72 := position, tokenIndex if !_rules[ruleCall]() { - goto l66 + goto l73 } - l67: + l74: { - position68, tokenIndex68 := position, tokenIndex + position75, tokenIndex75 := position, tokenIndex if !_rules[rulecomma]() { - goto l68 + goto l75 } if !_rules[ruleCall]() { - goto l68 + goto l75 } - goto l67 - l68: - position, tokenIndex = position68, tokenIndex68 + goto l74 + l75: + position, tokenIndex = position75, tokenIndex75 } { - position69, tokenIndex69 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex if !_rules[rulecomma]() { - goto l69 + goto l76 } if !_rules[ruleargs]() { - goto l69 + goto l76 } - goto l70 - l69: - position, tokenIndex = position69, tokenIndex69 + goto l77 + l76: + position, tokenIndex = position76, tokenIndex76 } - l70: - goto l65 - l66: - position, tokenIndex = position65, tokenIndex65 + l77: + goto l72 + l73: + position, tokenIndex = position72, tokenIndex72 if !_rules[ruleargs]() { - goto l71 + goto l78 } - goto l65 - l71: - position, tokenIndex = position65, tokenIndex65 + goto l72 + l78: + position, tokenIndex = position72, tokenIndex72 if !_rules[rulesp]() { - goto l63 + goto l70 } } - l65: - add(ruleallargs, position64) + l72: + add(ruleallargs, position71) } return true - l63: - position, tokenIndex = position63, tokenIndex63 + l70: + position, tokenIndex = position70, tokenIndex70 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position72, tokenIndex72 := position, tokenIndex + position79, tokenIndex79 := position, tokenIndex { - position73 := position + position80 := position if !_rules[rulearg]() { - goto l72 + goto l79 } { - position74, tokenIndex74 := position, tokenIndex + position81, tokenIndex81 := position, tokenIndex if !_rules[rulecomma]() { - goto l74 + goto l81 } if !_rules[ruleargs]() { - goto l74 + goto l81 } - goto l75 - l74: - position, tokenIndex = position74, tokenIndex74 + goto l82 + l81: + position, tokenIndex = position81, tokenIndex81 } - l75: + l82: if !_rules[rulesp]() { - goto l72 + goto l79 } - add(ruleargs, position73) + add(ruleargs, position80) } return true - l72: - position, tokenIndex = position72, tokenIndex72 + l79: + position, tokenIndex = position79, tokenIndex79 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position76, tokenIndex76 := position, tokenIndex + position83, tokenIndex83 := position, tokenIndex { - position77 := position + position84 := position { - position78, tokenIndex78 := position, tokenIndex + position85, tokenIndex85 := position, tokenIndex if !_rules[rulefield]() { - goto l79 + goto l86 } if !_rules[rulesp]() { - goto l79 + goto l86 } if buffer[position] != rune('=') { - goto l79 + goto l86 } position++ if !_rules[rulesp]() { - goto l79 + goto l86 } if !_rules[rulevalue]() { - goto l79 + goto l86 } - goto l78 - l79: - position, tokenIndex = position78, tokenIndex78 + goto l85 + l86: + position, tokenIndex = position85, tokenIndex85 if !_rules[rulefield]() { - goto l76 + goto l83 } if !_rules[rulesp]() { - goto l76 + goto l83 } { - position80 := position + position87 := position { - position81, tokenIndex81 := position, tokenIndex + position88, tokenIndex88 := position, tokenIndex if buffer[position] != rune('>') { - goto l82 + goto l89 } position++ if buffer[position] != rune('<') { - goto l82 + goto l89 } position++ { add(ruleAction14, position) } - goto l81 - l82: - position, tokenIndex = position81, tokenIndex81 + goto l88 + l89: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('<') { - goto l84 + goto l91 } position++ if buffer[position] != rune('=') { - goto l84 + goto l91 } position++ { add(ruleAction15, position) } - goto l81 - l84: - position, tokenIndex = position81, tokenIndex81 + goto l88 + l91: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('>') { - goto l86 + goto l93 } position++ if buffer[position] != rune('=') { - goto l86 + goto l93 } position++ { add(ruleAction16, position) } - goto l81 - l86: - position, tokenIndex = position81, tokenIndex81 + goto l88 + l93: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('=') { - goto l88 + goto l95 } position++ if buffer[position] != rune('=') { - goto l88 + goto l95 } position++ { add(ruleAction17, position) } - goto l81 - l88: - position, tokenIndex = position81, tokenIndex81 + goto l88 + l95: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('!') { - goto l90 + goto l97 } position++ if buffer[position] != rune('=') { - goto l90 + goto l97 } position++ { add(ruleAction18, position) } - goto l81 - l90: - position, tokenIndex = position81, tokenIndex81 + goto l88 + l97: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('<') { - goto l92 + goto l99 } position++ { add(ruleAction19, position) } - goto l81 - l92: - position, tokenIndex = position81, tokenIndex81 + goto l88 + l99: + position, tokenIndex = position88, tokenIndex88 if buffer[position] != rune('>') { - goto l76 + goto l83 } position++ { add(ruleAction20, position) } } - l81: - add(ruleCOND, position80) + l88: + add(ruleCOND, position87) } if !_rules[rulesp]() { - goto l76 + goto l83 } if !_rules[rulevalue]() { - goto l76 + goto l83 } } - l78: - add(rulearg, position77) + l85: + add(rulearg, position84) } return true - l76: - position, tokenIndex = position76, tokenIndex76 + l83: + position, tokenIndex = position83, tokenIndex83 return false }, /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ @@ -1365,102 +1425,102 @@ func (p *PQL) Init() { nil, /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action23)> */ func() bool { - position97, tokenIndex97 := position, tokenIndex + position104, tokenIndex104 := position, tokenIndex { - position98 := position + position105 := position { - position99 := position + position106 := position { - position100, tokenIndex100 := position, tokenIndex + position107, tokenIndex107 := position, tokenIndex { - position102, tokenIndex102 := position, tokenIndex + position109, tokenIndex109 := position, tokenIndex if buffer[position] != rune('-') { - goto l102 + goto l109 } position++ - goto l103 - l102: - position, tokenIndex = position102, tokenIndex102 + goto l110 + l109: + position, tokenIndex = position109, tokenIndex109 } - l103: + l110: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l101 + goto l108 } position++ - l104: + l111: { - position105, tokenIndex105 := position, tokenIndex + position112, tokenIndex112 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l105 + goto l112 } position++ - goto l104 - l105: - position, tokenIndex = position105, tokenIndex105 + goto l111 + l112: + position, tokenIndex = position112, tokenIndex112 } - goto l100 - l101: - position, tokenIndex = position100, tokenIndex100 + goto l107 + l108: + position, tokenIndex = position107, tokenIndex107 if buffer[position] != rune('0') { - goto l97 + goto l104 } position++ } - l100: - add(rulePegText, position99) + l107: + add(rulePegText, position106) } if !_rules[rulesp]() { - goto l97 + goto l104 } { add(ruleAction23, position) } - add(rulecondint, position98) + add(rulecondint, position105) } return true - l97: - position, tokenIndex = position97, tokenIndex97 + l104: + position, tokenIndex = position104, tokenIndex104 return false }, /* 8 condLT <- <(<(('<' '=') / '<')> sp Action24)> */ func() bool { - position107, tokenIndex107 := position, tokenIndex + position114, tokenIndex114 := position, tokenIndex { - position108 := position + position115 := position { - position109 := position + position116 := position { - position110, tokenIndex110 := position, tokenIndex + position117, tokenIndex117 := position, tokenIndex if buffer[position] != rune('<') { - goto l111 + goto l118 } position++ if buffer[position] != rune('=') { - goto l111 + goto l118 } position++ - goto l110 - l111: - position, tokenIndex = position110, tokenIndex110 + goto l117 + l118: + position, tokenIndex = position117, tokenIndex117 if buffer[position] != rune('<') { - goto l107 + goto l114 } position++ } - l110: - add(rulePegText, position109) + l117: + add(rulePegText, position116) } if !_rules[rulesp]() { - goto l107 + goto l114 } { add(ruleAction24, position) } - add(rulecondLT, position108) + add(rulecondLT, position115) } return true - l107: - position, tokenIndex = position107, tokenIndex107 + l114: + position, tokenIndex = position114, tokenIndex114 return false }, /* 9 condfield <- <( sp Action25)> */ @@ -1469,1381 +1529,1389 @@ func (p *PQL) Init() { nil, /* 11 value <- <(item / (lbrack Action28 list rbrack Action29))> */ func() bool { - position115, tokenIndex115 := position, tokenIndex + position122, tokenIndex122 := position, tokenIndex { - position116 := position + position123 := position { - position117, tokenIndex117 := position, tokenIndex + position124, tokenIndex124 := position, tokenIndex if !_rules[ruleitem]() { - goto l118 + goto l125 } - goto l117 - l118: - position, tokenIndex = position117, tokenIndex117 + goto l124 + l125: + position, tokenIndex = position124, tokenIndex124 { - position119 := position + position126 := position if buffer[position] != rune('[') { - goto l115 + goto l122 } position++ if !_rules[rulesp]() { - goto l115 + goto l122 } - add(rulelbrack, position119) + add(rulelbrack, position126) } { add(ruleAction28, position) } if !_rules[rulelist]() { - goto l115 + goto l122 } { - position121 := position + position128 := position if !_rules[rulesp]() { - goto l115 + goto l122 } if buffer[position] != rune(']') { - goto l115 + goto l122 } position++ if !_rules[rulesp]() { - goto l115 + goto l122 } - add(rulerbrack, position121) + add(rulerbrack, position128) } { add(ruleAction29, position) } } - l117: - add(rulevalue, position116) + l124: + add(rulevalue, position123) } return true - l115: - position, tokenIndex = position115, tokenIndex115 + l122: + position, tokenIndex = position122, tokenIndex122 return false }, /* 12 list <- <(item (comma list)?)> */ func() bool { - position123, tokenIndex123 := position, tokenIndex + position130, tokenIndex130 := position, tokenIndex { - position124 := position + position131 := position if !_rules[ruleitem]() { - goto l123 + goto l130 } { - position125, tokenIndex125 := position, tokenIndex + position132, tokenIndex132 := position, tokenIndex if !_rules[rulecomma]() { - goto l125 + goto l132 } if !_rules[rulelist]() { - goto l125 + goto l132 } - goto l126 - l125: - position, tokenIndex = position125, tokenIndex125 + goto l133 + l132: + position, tokenIndex = position132, tokenIndex132 } - l126: - add(rulelist, position124) + l133: + add(rulelist, position131) } return true - l123: - position, tokenIndex = position123, tokenIndex123 + l130: + position, tokenIndex = position130, tokenIndex130 return false }, /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action33) / (<('-'? '.' [0-9]+)> Action34) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action35) / ('"' '"' Action36) / ('\'' '\'' Action37))> */ func() bool { - position127, tokenIndex127 := position, tokenIndex + position134, tokenIndex134 := position, tokenIndex { - position128 := position + position135 := position { - position129, tokenIndex129 := position, tokenIndex + position136, tokenIndex136 := position, tokenIndex if buffer[position] != rune('n') { - goto l130 + goto l137 } position++ if buffer[position] != rune('u') { - goto l130 + goto l137 } position++ if buffer[position] != rune('l') { - goto l130 + goto l137 } position++ if buffer[position] != rune('l') { - goto l130 + goto l137 } position++ { - position131, tokenIndex131 := position, tokenIndex + position138, tokenIndex138 := position, tokenIndex { - position132, tokenIndex132 := position, tokenIndex + position139, tokenIndex139 := position, tokenIndex if !_rules[rulecomma]() { - goto l133 + goto l140 } - goto l132 - l133: - position, tokenIndex = position132, tokenIndex132 + goto l139 + l140: + position, tokenIndex = position139, tokenIndex139 if !_rules[rulesp]() { - goto l130 + goto l137 } if !_rules[ruleclose]() { - goto l130 + goto l137 } } - l132: - position, tokenIndex = position131, tokenIndex131 + l139: + position, tokenIndex = position138, tokenIndex138 } { add(ruleAction30, position) } - goto l129 - l130: - position, tokenIndex = position129, tokenIndex129 + goto l136 + l137: + position, tokenIndex = position136, tokenIndex136 if buffer[position] != rune('t') { - goto l135 + goto l142 } position++ if buffer[position] != rune('r') { - goto l135 + goto l142 } position++ if buffer[position] != rune('u') { - goto l135 + goto l142 } position++ if buffer[position] != rune('e') { - goto l135 + goto l142 } position++ { - position136, tokenIndex136 := position, tokenIndex + position143, tokenIndex143 := position, tokenIndex { - position137, tokenIndex137 := position, tokenIndex + position144, tokenIndex144 := position, tokenIndex if !_rules[rulecomma]() { - goto l138 + goto l145 } - goto l137 - l138: - position, tokenIndex = position137, tokenIndex137 + goto l144 + l145: + position, tokenIndex = position144, tokenIndex144 if !_rules[rulesp]() { - goto l135 + goto l142 } if !_rules[ruleclose]() { - goto l135 + goto l142 } } - l137: - position, tokenIndex = position136, tokenIndex136 + l144: + position, tokenIndex = position143, tokenIndex143 } { add(ruleAction31, position) } - goto l129 - l135: - position, tokenIndex = position129, tokenIndex129 + goto l136 + l142: + position, tokenIndex = position136, tokenIndex136 if buffer[position] != rune('f') { - goto l140 + goto l147 } position++ if buffer[position] != rune('a') { - goto l140 + goto l147 } position++ if buffer[position] != rune('l') { - goto l140 + goto l147 } position++ if buffer[position] != rune('s') { - goto l140 + goto l147 } position++ if buffer[position] != rune('e') { - goto l140 + goto l147 } position++ { - position141, tokenIndex141 := position, tokenIndex + position148, tokenIndex148 := position, tokenIndex { - position142, tokenIndex142 := position, tokenIndex + position149, tokenIndex149 := position, tokenIndex if !_rules[rulecomma]() { - goto l143 + goto l150 } - goto l142 - l143: - position, tokenIndex = position142, tokenIndex142 + goto l149 + l150: + position, tokenIndex = position149, tokenIndex149 if !_rules[rulesp]() { - goto l140 + goto l147 } if !_rules[ruleclose]() { - goto l140 + goto l147 } } - l142: - position, tokenIndex = position141, tokenIndex141 + l149: + position, tokenIndex = position148, tokenIndex148 } { add(ruleAction32, position) } - goto l129 - l140: - position, tokenIndex = position129, tokenIndex129 + goto l136 + l147: + position, tokenIndex = position136, tokenIndex136 { - position146 := position + position153 := position { - position147, tokenIndex147 := position, tokenIndex + position154, tokenIndex154 := position, tokenIndex if buffer[position] != rune('-') { - goto l147 + goto l154 } position++ - goto l148 - l147: - position, tokenIndex = position147, tokenIndex147 + goto l155 + l154: + position, tokenIndex = position154, tokenIndex154 } - l148: + l155: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l145 + goto l152 } position++ - l149: + l156: { - position150, tokenIndex150 := position, tokenIndex + position157, tokenIndex157 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l150 + goto l157 } position++ - goto l149 - l150: - position, tokenIndex = position150, tokenIndex150 + goto l156 + l157: + position, tokenIndex = position157, tokenIndex157 } - { - position151, tokenIndex151 := position, tokenIndex - if buffer[position] != rune('.') { - goto l151 - } - position++ - l153: - { - position154, tokenIndex154 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l154 - } - position++ - goto l153 - l154: - position, tokenIndex = position154, tokenIndex154 - } - goto l152 - l151: - position, tokenIndex = position151, tokenIndex151 - } - l152: - add(rulePegText, position146) - } - { - add(ruleAction33, position) - } - goto l129 - l145: - position, tokenIndex = position129, tokenIndex129 - { - position157 := position { position158, tokenIndex158 := position, tokenIndex - if buffer[position] != rune('-') { + if buffer[position] != rune('.') { goto l158 } position++ + l160: + { + position161, tokenIndex161 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l161 + } + position++ + goto l160 + l161: + position, tokenIndex = position161, tokenIndex161 + } goto l159 l158: position, tokenIndex = position158, tokenIndex158 } l159: - if buffer[position] != rune('.') { - goto l156 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l156 - } - position++ - l160: - { - position161, tokenIndex161 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l161 - } - position++ - goto l160 - l161: - position, tokenIndex = position161, tokenIndex161 - } - add(rulePegText, position157) + add(rulePegText, position153) } { - add(ruleAction34, position) + add(ruleAction33, position) } - goto l129 - l156: - position, tokenIndex = position129, tokenIndex129 + goto l136 + l152: + position, tokenIndex = position136, tokenIndex136 { position164 := position { - position167, tokenIndex167 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { + position165, tokenIndex165 := position, tokenIndex + if buffer[position] != rune('-') { + goto l165 + } + position++ + goto l166 + l165: + position, tokenIndex = position165, tokenIndex165 + } + l166: + if buffer[position] != rune('.') { + goto l163 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l163 + } + position++ + l167: + { + position168, tokenIndex168 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l168 } position++ goto l167 l168: - position, tokenIndex = position167, tokenIndex167 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l169 - } - position++ - goto l167 - l169: - position, tokenIndex = position167, tokenIndex167 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l170 - } - position++ - goto l167 - l170: - position, tokenIndex = position167, tokenIndex167 - if buffer[position] != rune('-') { - goto l171 - } - position++ - goto l167 - l171: - position, tokenIndex = position167, tokenIndex167 - if buffer[position] != rune('_') { - goto l172 - } - position++ - goto l167 - l172: - position, tokenIndex = position167, tokenIndex167 - if buffer[position] != rune(':') { - goto l163 - } - position++ - } - l167: - l165: - { - position166, tokenIndex166 := position, tokenIndex - { - position173, tokenIndex173 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l174 - } - position++ - goto l173 - l174: - position, tokenIndex = position173, tokenIndex173 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l175 - } - position++ - goto l173 - l175: - position, tokenIndex = position173, tokenIndex173 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l176 - } - position++ - goto l173 - l176: - position, tokenIndex = position173, tokenIndex173 - if buffer[position] != rune('-') { - goto l177 - } - position++ - goto l173 - l177: - position, tokenIndex = position173, tokenIndex173 - if buffer[position] != rune('_') { - goto l178 - } - position++ - goto l173 - l178: - position, tokenIndex = position173, tokenIndex173 - if buffer[position] != rune(':') { - goto l166 - } - position++ - } - l173: - goto l165 - l166: - position, tokenIndex = position166, tokenIndex166 + position, tokenIndex = position168, tokenIndex168 } add(rulePegText, position164) } + { + add(ruleAction34, position) + } + goto l136 + l163: + position, tokenIndex = position136, tokenIndex136 + { + position171 := position + { + position174, tokenIndex174 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l175 + } + position++ + goto l174 + l175: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l176 + } + position++ + goto l174 + l176: + position, tokenIndex = position174, tokenIndex174 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l177 + } + position++ + goto l174 + l177: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('-') { + goto l178 + } + position++ + goto l174 + l178: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune('_') { + goto l179 + } + position++ + goto l174 + l179: + position, tokenIndex = position174, tokenIndex174 + if buffer[position] != rune(':') { + goto l170 + } + position++ + } + l174: + l172: + { + position173, tokenIndex173 := position, tokenIndex + { + position180, tokenIndex180 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l181 + } + position++ + goto l180 + l181: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l182 + } + position++ + goto l180 + l182: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l183 + } + position++ + goto l180 + l183: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('-') { + goto l184 + } + position++ + goto l180 + l184: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('_') { + goto l185 + } + position++ + goto l180 + l185: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune(':') { + goto l173 + } + position++ + } + l180: + goto l172 + l173: + position, tokenIndex = position173, tokenIndex173 + } + add(rulePegText, position171) + } { add(ruleAction35, position) } - goto l129 - l163: - position, tokenIndex = position129, tokenIndex129 + goto l136 + l170: + position, tokenIndex = position136, tokenIndex136 if buffer[position] != rune('"') { - goto l180 + goto l187 } position++ { - position181 := position + position188 := position if !_rules[ruledoublequotedstring]() { - goto l180 + goto l187 } - add(rulePegText, position181) + add(rulePegText, position188) } if buffer[position] != rune('"') { - goto l180 + goto l187 } position++ { add(ruleAction36, position) } - goto l129 - l180: - position, tokenIndex = position129, tokenIndex129 + goto l136 + l187: + position, tokenIndex = position136, tokenIndex136 if buffer[position] != rune('\'') { - goto l127 + goto l134 } position++ { - position183 := position + position190 := position if !_rules[rulesinglequotedstring]() { - goto l127 + goto l134 } - add(rulePegText, position183) + add(rulePegText, position190) } if buffer[position] != rune('\'') { - goto l127 + goto l134 } position++ { add(ruleAction37, position) } } - l129: - add(ruleitem, position128) + l136: + add(ruleitem, position135) } return true - l127: - position, tokenIndex = position127, tokenIndex127 + l134: + position, tokenIndex = position134, tokenIndex134 return false }, /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position186 := position - l187: + position193 := position + l194: { - position188, tokenIndex188 := position, tokenIndex + position195, tokenIndex195 := position, tokenIndex { - position189, tokenIndex189 := position, tokenIndex + position196, tokenIndex196 := position, tokenIndex { - position191, tokenIndex191 := position, tokenIndex + position198, tokenIndex198 := position, tokenIndex { - position192, tokenIndex192 := position, tokenIndex + position199, tokenIndex199 := position, tokenIndex if buffer[position] != rune('"') { - goto l193 + goto l200 } position++ - goto l192 - l193: - position, tokenIndex = position192, tokenIndex192 + goto l199 + l200: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\\') { - goto l194 + goto l201 } position++ - goto l192 - l194: - position, tokenIndex = position192, tokenIndex192 + goto l199 + l201: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\n') { - goto l191 + goto l198 } position++ } - l192: - goto l190 - l191: - position, tokenIndex = position191, tokenIndex191 + l199: + goto l197 + l198: + position, tokenIndex = position198, tokenIndex198 } if !matchDot() { - goto l190 + goto l197 } - goto l189 - l190: - position, tokenIndex = position189, tokenIndex189 + goto l196 + l197: + position, tokenIndex = position196, tokenIndex196 if buffer[position] != rune('\\') { - goto l195 + goto l202 } position++ if buffer[position] != rune('n') { - goto l195 + goto l202 } position++ - goto l189 - l195: - position, tokenIndex = position189, tokenIndex189 + goto l196 + l202: + position, tokenIndex = position196, tokenIndex196 if buffer[position] != rune('\\') { - goto l196 + goto l203 } position++ if buffer[position] != rune('"') { - goto l196 + goto l203 } position++ - goto l189 - l196: - position, tokenIndex = position189, tokenIndex189 + goto l196 + l203: + position, tokenIndex = position196, tokenIndex196 if buffer[position] != rune('\\') { - goto l197 + goto l204 } position++ if buffer[position] != rune('\'') { - goto l197 + goto l204 } position++ - goto l189 - l197: - position, tokenIndex = position189, tokenIndex189 + goto l196 + l204: + position, tokenIndex = position196, tokenIndex196 if buffer[position] != rune('\\') { - goto l188 + goto l195 } position++ if buffer[position] != rune('\\') { - goto l188 + goto l195 } position++ } - l189: - goto l187 - l188: - position, tokenIndex = position188, tokenIndex188 + l196: + goto l194 + l195: + position, tokenIndex = position195, tokenIndex195 } - add(ruledoublequotedstring, position186) + add(ruledoublequotedstring, position193) } return true }, /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position199 := position - l200: + position206 := position + l207: { - position201, tokenIndex201 := position, tokenIndex + position208, tokenIndex208 := position, tokenIndex { - position202, tokenIndex202 := position, tokenIndex + position209, tokenIndex209 := position, tokenIndex { - position204, tokenIndex204 := position, tokenIndex + position211, tokenIndex211 := position, tokenIndex { - position205, tokenIndex205 := position, tokenIndex + position212, tokenIndex212 := position, tokenIndex if buffer[position] != rune('\'') { - goto l206 + goto l213 } position++ - goto l205 - l206: - position, tokenIndex = position205, tokenIndex205 + goto l212 + l213: + position, tokenIndex = position212, tokenIndex212 if buffer[position] != rune('\\') { - goto l207 + goto l214 } position++ - goto l205 - l207: - position, tokenIndex = position205, tokenIndex205 + goto l212 + l214: + position, tokenIndex = position212, tokenIndex212 if buffer[position] != rune('\n') { - goto l204 + goto l211 } position++ } - l205: - goto l203 - l204: - position, tokenIndex = position204, tokenIndex204 + l212: + goto l210 + l211: + position, tokenIndex = position211, tokenIndex211 } if !matchDot() { - goto l203 + goto l210 } - goto l202 - l203: - position, tokenIndex = position202, tokenIndex202 + goto l209 + l210: + position, tokenIndex = position209, tokenIndex209 if buffer[position] != rune('\\') { - goto l208 + goto l215 } position++ if buffer[position] != rune('n') { - goto l208 + goto l215 } position++ - goto l202 - l208: - position, tokenIndex = position202, tokenIndex202 + goto l209 + l215: + position, tokenIndex = position209, tokenIndex209 if buffer[position] != rune('\\') { - goto l209 + goto l216 } position++ if buffer[position] != rune('"') { - goto l209 + goto l216 } position++ - goto l202 - l209: - position, tokenIndex = position202, tokenIndex202 + goto l209 + l216: + position, tokenIndex = position209, tokenIndex209 if buffer[position] != rune('\\') { - goto l210 + goto l217 } position++ if buffer[position] != rune('\'') { - goto l210 + goto l217 } position++ - goto l202 - l210: - position, tokenIndex = position202, tokenIndex202 + goto l209 + l217: + position, tokenIndex = position209, tokenIndex209 if buffer[position] != rune('\\') { - goto l201 + goto l208 } position++ if buffer[position] != rune('\\') { - goto l201 + goto l208 } position++ } - l202: - goto l200 - l201: - position, tokenIndex = position201, tokenIndex201 + l209: + goto l207 + l208: + position, tokenIndex = position208, tokenIndex208 } - add(rulesinglequotedstring, position199) + add(rulesinglequotedstring, position206) } return true }, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position211, tokenIndex211 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex { - position212 := position + position219 := position { - position213, tokenIndex213 := position, tokenIndex + position220, tokenIndex220 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l214 + goto l221 } position++ - goto l213 - l214: - position, tokenIndex = position213, tokenIndex213 + goto l220 + l221: + position, tokenIndex = position220, tokenIndex220 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l211 + goto l218 } position++ } - l213: - l215: + l220: + l222: { - position216, tokenIndex216 := position, tokenIndex + position223, tokenIndex223 := position, tokenIndex { - position217, tokenIndex217 := position, tokenIndex + position224, tokenIndex224 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l218 + goto l225 } position++ - goto l217 - l218: - position, tokenIndex = position217, tokenIndex217 + goto l224 + l225: + position, tokenIndex = position224, tokenIndex224 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l219 + goto l226 } position++ - goto l217 - l219: - position, tokenIndex = position217, tokenIndex217 + goto l224 + l226: + position, tokenIndex = position224, tokenIndex224 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l220 + goto l227 } position++ - goto l217 - l220: - position, tokenIndex = position217, tokenIndex217 + goto l224 + l227: + position, tokenIndex = position224, tokenIndex224 if buffer[position] != rune('_') { - goto l221 + goto l228 } position++ - goto l217 - l221: - position, tokenIndex = position217, tokenIndex217 + goto l224 + l228: + position, tokenIndex = position224, tokenIndex224 if buffer[position] != rune('-') { - goto l216 + goto l223 } position++ } - l217: - goto l215 - l216: - position, tokenIndex = position216, tokenIndex216 + l224: + goto l222 + l223: + position, tokenIndex = position223, tokenIndex223 } - add(rulefieldExpr, position212) + add(rulefieldExpr, position219) } return true - l211: - position, tokenIndex = position211, tokenIndex211 + l218: + position, tokenIndex = position218, tokenIndex218 return false }, /* 17 field <- <(<(fieldExpr / reserved)> Action38)> */ func() bool { - position222, tokenIndex222 := position, tokenIndex + position229, tokenIndex229 := position, tokenIndex { - position223 := position + position230 := position { - position224 := position + position231 := position { - position225, tokenIndex225 := position, tokenIndex + position232, tokenIndex232 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l226 + goto l233 } - goto l225 - l226: - position, tokenIndex = position225, tokenIndex225 + goto l232 + l233: + position, tokenIndex = position232, tokenIndex232 { - position227 := position + position234 := position { - position228, tokenIndex228 := position, tokenIndex + position235, tokenIndex235 := position, tokenIndex if buffer[position] != rune('_') { - goto l229 + goto l236 } position++ if buffer[position] != rune('r') { - goto l229 + goto l236 } position++ if buffer[position] != rune('o') { - goto l229 + goto l236 } position++ if buffer[position] != rune('w') { - goto l229 + goto l236 } position++ - goto l228 - l229: - position, tokenIndex = position228, tokenIndex228 + goto l235 + l236: + position, tokenIndex = position235, tokenIndex235 if buffer[position] != rune('_') { - goto l230 + goto l237 } position++ if buffer[position] != rune('c') { - goto l230 + goto l237 } position++ if buffer[position] != rune('o') { - goto l230 + goto l237 } position++ if buffer[position] != rune('l') { - goto l230 + goto l237 } position++ - goto l228 - l230: - position, tokenIndex = position228, tokenIndex228 + goto l235 + l237: + position, tokenIndex = position235, tokenIndex235 if buffer[position] != rune('_') { - goto l231 + goto l238 } position++ if buffer[position] != rune('s') { - goto l231 + goto l238 } position++ if buffer[position] != rune('t') { - goto l231 + goto l238 } position++ if buffer[position] != rune('a') { - goto l231 + goto l238 } position++ if buffer[position] != rune('r') { - goto l231 + goto l238 } position++ if buffer[position] != rune('t') { - goto l231 + goto l238 } position++ - goto l228 - l231: - position, tokenIndex = position228, tokenIndex228 + goto l235 + l238: + position, tokenIndex = position235, tokenIndex235 if buffer[position] != rune('_') { - goto l232 + goto l239 } position++ if buffer[position] != rune('e') { - goto l232 + goto l239 } position++ if buffer[position] != rune('n') { - goto l232 + goto l239 } position++ if buffer[position] != rune('d') { - goto l232 + goto l239 } position++ - goto l228 - l232: - position, tokenIndex = position228, tokenIndex228 + goto l235 + l239: + position, tokenIndex = position235, tokenIndex235 if buffer[position] != rune('_') { - goto l233 + goto l240 } position++ if buffer[position] != rune('t') { - goto l233 + goto l240 } position++ if buffer[position] != rune('i') { - goto l233 + goto l240 } position++ if buffer[position] != rune('m') { - goto l233 + goto l240 } position++ if buffer[position] != rune('e') { - goto l233 + goto l240 } position++ if buffer[position] != rune('s') { - goto l233 + goto l240 } position++ if buffer[position] != rune('t') { - goto l233 + goto l240 } position++ if buffer[position] != rune('a') { - goto l233 + goto l240 } position++ if buffer[position] != rune('m') { - goto l233 + goto l240 } position++ if buffer[position] != rune('p') { - goto l233 + goto l240 } position++ - goto l228 - l233: - position, tokenIndex = position228, tokenIndex228 + goto l235 + l240: + position, tokenIndex = position235, tokenIndex235 if buffer[position] != rune('_') { - goto l222 + goto l229 } position++ if buffer[position] != rune('f') { - goto l222 + goto l229 } position++ if buffer[position] != rune('i') { - goto l222 + goto l229 } position++ if buffer[position] != rune('e') { - goto l222 + goto l229 } position++ if buffer[position] != rune('l') { - goto l222 + goto l229 } position++ if buffer[position] != rune('d') { - goto l222 + goto l229 } position++ } - l228: - add(rulereserved, position227) + l235: + add(rulereserved, position234) } } - l225: - add(rulePegText, position224) + l232: + add(rulePegText, position231) } { add(ruleAction38, position) } - add(rulefield, position223) + add(rulefield, position230) } return true - l222: - position, tokenIndex = position222, tokenIndex222 + l229: + position, tokenIndex = position229, tokenIndex229 return false }, /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, /* 19 posfield <- <( Action39)> */ func() bool { - position236, tokenIndex236 := position, tokenIndex + position243, tokenIndex243 := position, tokenIndex { - position237 := position + position244 := position { - position238 := position + position245 := position if !_rules[rulefieldExpr]() { - goto l236 + goto l243 } - add(rulePegText, position238) + add(rulePegText, position245) } { add(ruleAction39, position) } - add(ruleposfield, position237) + add(ruleposfield, position244) } return true - l236: - position, tokenIndex = position236, tokenIndex236 + l243: + position, tokenIndex = position243, tokenIndex243 return false }, /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ - func() bool { - position240, tokenIndex240 := position, tokenIndex - { - position241 := position - { - position242, tokenIndex242 := position, tokenIndex - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l243 - } - position++ - l244: - { - position245, tokenIndex245 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l245 - } - position++ - goto l244 - l245: - position, tokenIndex = position245, tokenIndex245 - } - goto l242 - l243: - position, tokenIndex = position242, tokenIndex242 - if buffer[position] != rune('0') { - goto l240 - } - position++ - } - l242: - add(ruleuint, position241) - } - return true - l240: - position, tokenIndex = position240, tokenIndex240 - return false - }, - /* 21 uintrow <- <( Action40)> */ - nil, - /* 22 col <- <(( Action41) / ('\'' '\'' Action42) / ('"' '"' Action43))> */ func() bool { position247, tokenIndex247 := position, tokenIndex { position248 := position { position249, tokenIndex249 := position, tokenIndex - { - position251 := position - if !_rules[ruleuint]() { - goto l250 - } - add(rulePegText, position251) + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l250 } + position++ + l251: { - add(ruleAction41, position) + position252, tokenIndex252 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l252 + } + position++ + goto l251 + l252: + position, tokenIndex = position252, tokenIndex252 } goto l249 l250: position, tokenIndex = position249, tokenIndex249 - if buffer[position] != rune('\'') { - goto l253 - } - position++ - { - position254 := position - if !_rules[rulesinglequotedstring]() { - goto l253 - } - add(rulePegText, position254) - } - if buffer[position] != rune('\'') { - goto l253 - } - position++ - { - add(ruleAction42, position) - } - goto l249 - l253: - position, tokenIndex = position249, tokenIndex249 - if buffer[position] != rune('"') { + if buffer[position] != rune('0') { goto l247 } position++ - { - position256 := position - if !_rules[ruledoublequotedstring]() { - goto l247 - } - add(rulePegText, position256) - } - if buffer[position] != rune('"') { - goto l247 - } - position++ - { - add(ruleAction43, position) - } } l249: - add(rulecol, position248) + add(ruleuint, position248) } return true l247: position, tokenIndex = position247, tokenIndex247 return false }, - /* 23 open <- <('(' sp)> */ + /* 21 uintrow <- <( Action40)> */ + nil, + /* 22 col <- <(( Action41) / ('\'' '\'' Action42) / ('"' '"' Action43))> */ func() bool { - position258, tokenIndex258 := position, tokenIndex + position254, tokenIndex254 := position, tokenIndex { - position259 := position - if buffer[position] != rune('(') { - goto l258 - } - position++ - if !_rules[rulesp]() { - goto l258 - } - add(ruleopen, position259) - } - return true - l258: - position, tokenIndex = position258, tokenIndex258 - return false - }, - /* 24 close <- <(')' sp)> */ - func() bool { - position260, tokenIndex260 := position, tokenIndex - { - position261 := position - if buffer[position] != rune(')') { - goto l260 - } - position++ - if !_rules[rulesp]() { - goto l260 - } - add(ruleclose, position261) - } - return true - l260: - position, tokenIndex = position260, tokenIndex260 - return false - }, - /* 25 sp <- <(' ' / '\t' / '\n')*> */ - func() bool { - { - position263 := position - l264: + position255 := position { - position265, tokenIndex265 := position, tokenIndex + position256, tokenIndex256 := position, tokenIndex { - position266, tokenIndex266 := position, tokenIndex + position258 := position + if !_rules[ruleuint]() { + goto l257 + } + add(rulePegText, position258) + } + { + add(ruleAction41, position) + } + goto l256 + l257: + position, tokenIndex = position256, tokenIndex256 + if buffer[position] != rune('\'') { + goto l260 + } + position++ + { + position261 := position + if !_rules[rulesinglequotedstring]() { + goto l260 + } + add(rulePegText, position261) + } + if buffer[position] != rune('\'') { + goto l260 + } + position++ + { + add(ruleAction42, position) + } + goto l256 + l260: + position, tokenIndex = position256, tokenIndex256 + if buffer[position] != rune('"') { + goto l254 + } + position++ + { + position263 := position + if !_rules[ruledoublequotedstring]() { + goto l254 + } + add(rulePegText, position263) + } + if buffer[position] != rune('"') { + goto l254 + } + position++ + { + add(ruleAction43, position) + } + } + l256: + add(rulecol, position255) + } + return true + l254: + position, tokenIndex = position254, tokenIndex254 + return false + }, + /* 23 row <- <(( Action44) / ('\'' '\'' Action45) / ('"' '"' Action46))> */ + nil, + /* 24 open <- <('(' sp)> */ + func() bool { + position266, tokenIndex266 := position, tokenIndex + { + position267 := position + if buffer[position] != rune('(') { + goto l266 + } + position++ + if !_rules[rulesp]() { + goto l266 + } + add(ruleopen, position267) + } + return true + l266: + position, tokenIndex = position266, tokenIndex266 + return false + }, + /* 25 close <- <(')' sp)> */ + func() bool { + position268, tokenIndex268 := position, tokenIndex + { + position269 := position + if buffer[position] != rune(')') { + goto l268 + } + position++ + if !_rules[rulesp]() { + goto l268 + } + add(ruleclose, position269) + } + return true + l268: + position, tokenIndex = position268, tokenIndex268 + return false + }, + /* 26 sp <- <(' ' / '\t' / '\n')*> */ + func() bool { + { + position271 := position + l272: + { + position273, tokenIndex273 := position, tokenIndex + { + position274, tokenIndex274 := position, tokenIndex if buffer[position] != rune(' ') { - goto l267 + goto l275 } position++ - goto l266 - l267: - position, tokenIndex = position266, tokenIndex266 + goto l274 + l275: + position, tokenIndex = position274, tokenIndex274 if buffer[position] != rune('\t') { - goto l268 + goto l276 } position++ - goto l266 - l268: - position, tokenIndex = position266, tokenIndex266 + goto l274 + l276: + position, tokenIndex = position274, tokenIndex274 if buffer[position] != rune('\n') { - goto l265 + goto l273 } position++ } - l266: - goto l264 - l265: - position, tokenIndex = position265, tokenIndex265 + l274: + goto l272 + l273: + position, tokenIndex = position273, tokenIndex273 } - add(rulesp, position263) + add(rulesp, position271) } return true }, - /* 26 comma <- <(sp ',' sp)> */ + /* 27 comma <- <(sp ',' sp)> */ func() bool { - position269, tokenIndex269 := position, tokenIndex + position277, tokenIndex277 := position, tokenIndex { - position270 := position + position278 := position if !_rules[rulesp]() { - goto l269 + goto l277 } if buffer[position] != rune(',') { - goto l269 + goto l277 } position++ if !_rules[rulesp]() { - goto l269 + goto l277 } - add(rulecomma, position270) + add(rulecomma, position278) } return true - l269: - position, tokenIndex = position269, tokenIndex269 + l277: + position, tokenIndex = position277, tokenIndex277 return false }, - /* 27 lbrack <- <('[' sp)> */ + /* 28 lbrack <- <('[' sp)> */ nil, - /* 28 rbrack <- <(sp ']' sp)> */ + /* 29 rbrack <- <(sp ']' sp)> */ nil, - /* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 30 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ nil, - /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + /* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position274, tokenIndex274 := position, tokenIndex + position282, tokenIndex282 := position, tokenIndex { - position275 := position + position283 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if buffer[position] != rune('-') { - goto l274 + goto l282 } position++ { - position276, tokenIndex276 := position, tokenIndex + position284, tokenIndex284 := position, tokenIndex if buffer[position] != rune('0') { - goto l277 + goto l285 } position++ - goto l276 - l277: - position, tokenIndex = position276, tokenIndex276 + goto l284 + l285: + position, tokenIndex = position284, tokenIndex284 if buffer[position] != rune('1') { - goto l274 + goto l282 } position++ } - l276: + l284: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if buffer[position] != rune('-') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if buffer[position] != rune('T') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if buffer[position] != rune(':') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l274 + goto l282 } position++ - add(ruletimestampbasicfmt, position275) + add(ruletimestampbasicfmt, position283) } return true - l274: - position, tokenIndex = position274, tokenIndex274 + l282: + position, tokenIndex = position282, tokenIndex282 return false }, - /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ + /* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position278, tokenIndex278 := position, tokenIndex + position286, tokenIndex286 := position, tokenIndex { - position279 := position + position287 := position { - position280, tokenIndex280 := position, tokenIndex + position288, tokenIndex288 := position, tokenIndex if buffer[position] != rune('"') { - goto l281 + goto l289 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l281 + goto l289 } if buffer[position] != rune('"') { - goto l281 + goto l289 } position++ - goto l280 - l281: - position, tokenIndex = position280, tokenIndex280 + goto l288 + l289: + position, tokenIndex = position288, tokenIndex288 if buffer[position] != rune('\'') { - goto l282 + goto l290 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l282 + goto l290 } if buffer[position] != rune('\'') { - goto l282 + goto l290 } position++ - goto l280 - l282: - position, tokenIndex = position280, tokenIndex280 + goto l288 + l290: + position, tokenIndex = position288, tokenIndex288 if !_rules[ruletimestampbasicfmt]() { - goto l278 + goto l286 } } - l280: - add(ruletimestampfmt, position279) + l288: + add(ruletimestampfmt, position287) } return true - l278: - position, tokenIndex = position278, tokenIndex278 + l286: + position, tokenIndex = position286, tokenIndex286 return false }, - /* 32 timestamp <- <( Action44)> */ + /* 33 timestamp <- <( Action47)> */ nil, - /* 34 Action0 <- <{p.startCall("Set")}> */ + /* 35 Action0 <- <{p.startCall("Set")}> */ nil, - /* 35 Action1 <- <{p.endCall()}> */ + /* 36 Action1 <- <{p.endCall()}> */ nil, - /* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */ + /* 37 Action2 <- <{p.startCall("SetRowAttrs")}> */ nil, - /* 37 Action3 <- <{p.endCall()}> */ + /* 38 Action3 <- <{p.endCall()}> */ nil, - /* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */ + /* 39 Action4 <- <{p.startCall("SetColumnAttrs")}> */ nil, - /* 39 Action5 <- <{p.endCall()}> */ + /* 40 Action5 <- <{p.endCall()}> */ nil, - /* 40 Action6 <- <{p.startCall("Clear")}> */ + /* 41 Action6 <- <{p.startCall("Clear")}> */ nil, - /* 41 Action7 <- <{p.endCall()}> */ + /* 42 Action7 <- <{p.endCall()}> */ nil, - /* 42 Action8 <- <{p.startCall("TopN")}> */ + /* 43 Action8 <- <{p.startCall("TopN")}> */ nil, - /* 43 Action9 <- <{p.endCall()}> */ + /* 44 Action9 <- <{p.endCall()}> */ nil, - /* 44 Action10 <- <{p.startCall("Range")}> */ + /* 45 Action10 <- <{p.startCall("Range")}> */ nil, - /* 45 Action11 <- <{p.endCall()}> */ + /* 46 Action11 <- <{p.endCall()}> */ nil, nil, - /* 47 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 48 Action13 <- <{ p.endCall() }> */ + /* 49 Action13 <- <{ p.endCall() }> */ nil, - /* 49 Action14 <- <{ p.addBTWN() }> */ + /* 50 Action14 <- <{ p.addBTWN() }> */ nil, - /* 50 Action15 <- <{ p.addLTE() }> */ + /* 51 Action15 <- <{ p.addLTE() }> */ nil, - /* 51 Action16 <- <{ p.addGTE() }> */ + /* 52 Action16 <- <{ p.addGTE() }> */ nil, - /* 52 Action17 <- <{ p.addEQ() }> */ + /* 53 Action17 <- <{ p.addEQ() }> */ nil, - /* 53 Action18 <- <{ p.addNEQ() }> */ + /* 54 Action18 <- <{ p.addNEQ() }> */ nil, - /* 54 Action19 <- <{ p.addLT() }> */ + /* 55 Action19 <- <{ p.addLT() }> */ nil, - /* 55 Action20 <- <{ p.addGT() }> */ + /* 56 Action20 <- <{ p.addGT() }> */ nil, - /* 56 Action21 <- <{p.startConditional()}> */ + /* 57 Action21 <- <{p.startConditional()}> */ nil, - /* 57 Action22 <- <{p.endConditional()}> */ + /* 58 Action22 <- <{p.endConditional()}> */ nil, - /* 58 Action23 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action23 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 59 Action24 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action24 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 60 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 61 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 62 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 62 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 63 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 63 Action28 <- <{ p.startList() }> */ + /* 64 Action28 <- <{ p.startList() }> */ nil, - /* 64 Action29 <- <{ p.endList() }> */ + /* 65 Action29 <- <{ p.endList() }> */ nil, - /* 65 Action30 <- <{ p.addVal(nil) }> */ + /* 66 Action30 <- <{ p.addVal(nil) }> */ nil, - /* 66 Action31 <- <{ p.addVal(true) }> */ + /* 67 Action31 <- <{ p.addVal(true) }> */ nil, - /* 67 Action32 <- <{ p.addVal(false) }> */ + /* 68 Action32 <- <{ p.addVal(false) }> */ nil, - /* 68 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 69 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 70 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 70 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 71 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 71 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 72 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 72 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 73 Action38 <- <{ p.addField(buffer[begin:end]) }> */ + /* 74 Action38 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 74 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 75 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 75 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 76 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 76 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 77 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 77 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 78 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 78 Action43 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 79 Action43 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 79 Action44 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 80 Action44 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + nil, + /* 81 Action45 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 82 Action46 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 83 Action47 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 076261599..a9bb00e59 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -160,6 +160,14 @@ func TestPEGWorking(t *testing.T) { name: "SetRowAttrs2args", input: "SetRowAttrs(blah, 9, a=47, b=bval)", ncalls: 1}, + { + name: "SetRowAttrsWithRowKeySingleQuote", + input: "SetRowAttrs(blah, 'rowKey', a=47)", + ncalls: 1}, + { + name: "SetRowAttrsWithRowKeyDoubleQuote", + input: `SetRowAttrs(blah, "rowKey", a=47)`, + ncalls: 1}, { name: "SetColumnAttrs", input: "SetColumnAttrs(9, a=47)", @@ -168,6 +176,14 @@ func TestPEGWorking(t *testing.T) { name: "SetColumnAttrs2args", input: "SetColumnAttrs(9, a=47, b=bval)", ncalls: 1}, + { + name: "SetColumnAttrsWithColKeySingleQuote", + input: "SetColumnAttrs('colKey', a=47)", + ncalls: 1}, + { + name: "SetColumnAttrsWithColKeyDoubleQuote", + input: `SetColumnAttrs("colKey", a=47)`, + ncalls: 1}, { name: "Clear", input: "Clear(1, a=53)", @@ -330,6 +346,28 @@ func TestPQLDeepEquality(t *testing.T) { "_row": int64(9), }, }}, + { + name: "SetRowAttrsWithRowKeySingleQuote", + call: "SetRowAttrs(myfield, 'rowKey', z=4)", + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": "rowKey", + }, + }}, + { + name: "SetRowAttrsWithRowKeyDoubleQuote", + call: `SetRowAttrs(myfield, "rowKey", z=4)`, + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_field": "myfield", + "_row": "rowKey", + }, + }}, { name: "SetColumnAttrs", call: "SetColumnAttrs(9, z=4)", @@ -340,6 +378,26 @@ func TestPQLDeepEquality(t *testing.T) { "_col": int64(9), }, }}, + { + name: "SetColumnAttrsWithColKeySingleQuote", + call: "SetColumnAttrs('colKey', z=4)", + exp: &Call{ + Name: "SetColumnAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_col": "colKey", + }, + }}, + { + name: "SetColumnAttrsWithColKeyDoubleQuote", + call: `SetColumnAttrs("colKey", z=4)`, + exp: &Call{ + Name: "SetColumnAttrs", + Args: map[string]interface{}{ + "z": int64(4), + "_col": "colKey", + }, + }}, { name: "Clear", call: "Clear(1, a=7)", diff --git a/test/index.go b/test/index.go index ae2b997a9..066ea615f 100644 --- a/test/index.go +++ b/test/index.go @@ -83,8 +83,8 @@ func (i *Index) CreateField(name string, opts ...pilosa.FieldOption) (*Field, er } // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. -func (i *Index) CreateFieldIfNotExists(name string, opts pilosa.FieldOption) (*Field, error) { - f, err := i.Index.CreateFieldIfNotExists(name, opts) +func (i *Index) CreateFieldIfNotExists(name string, opts ...pilosa.FieldOption) (*Field, error) { + f, err := i.Index.CreateFieldIfNotExists(name, opts...) if err != nil { return nil, err } From 13c201d768c4409d16fcf8b2af3e7f704eff8435 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 27 Jul 2018 11:05:23 -0500 Subject: [PATCH 06/17] remove errant spew.Dump() --- executor_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/executor_test.go b/executor_test.go index 05c99bfd1..1bcb17524 100644 --- a/executor_test.go +++ b/executor_test.go @@ -512,12 +512,6 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { t.Fatal(err) } - result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}) - if err != nil { - t.Fatal(err) - } - spew.Dump(result) - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}); err != nil { t.Fatal(err) } else if attrs := result.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { From 5410dcd6b1cdfd0557ba1206fb00c65b9f55dc53 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 24 Jul 2018 10:06:20 -0500 Subject: [PATCH 07/17] Fix errant references of server.primaryTranslateStore to server.translateFile --- api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index afb0e2379..b29b3c1fc 100644 --- a/api.go +++ b/api.go @@ -135,9 +135,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er } // Translate column attributes, if necessary. - if api.server.primaryTranslateStore != nil { + if api.server.translateFile != nil { for _, col := range resp.ColumnAttrSets { - v, err := api.server.primaryTranslateStore.TranslateColumnToString(req.Index, col.ID) + v, err := api.server.translateFile.TranslateColumnToString(req.Index, col.ID) if err != nil { return resp, err } @@ -760,7 +760,7 @@ func (api *API) ResizeAbort() error { const translateStoreBufferSize = 65536 func (api *API) GetTranslateData(ctx context.Context, w io.WriteCloser, offset int64) error { - rc, err := api.server.primaryTranslateStore.Reader(ctx, offset) + rc, err := api.server.translateFile.Reader(ctx, offset) if err != nil { return errors.Wrap(err, "read from translate store") } From 3ffafaca4aa4bb29a9fae2155cb85b40135025be Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 26 Jul 2018 14:55:07 -0500 Subject: [PATCH 08/17] ignore remote translation --- api.go | 37 ++++----------------------------- executor.go | 20 +++++++++++------- http/handler.go | 31 ++++++++++++++++++++++----- http/translator.go | 2 +- mock/translator.go | 12 +++++------ server/config.go | 2 +- translate.go | 52 +++++++++++++++++++++++----------------------- translate_test.go | 42 ++++++++++++++++++------------------- 8 files changed, 98 insertions(+), 100 deletions(-) diff --git a/api.go b/api.go index b29b3c1fc..649a99cd2 100644 --- a/api.go +++ b/api.go @@ -756,46 +756,17 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "complete current job") } -// translateStoreBufferSize is the buffer size used for streaming data. -const translateStoreBufferSize = 65536 - -func (api *API) GetTranslateData(ctx context.Context, w io.WriteCloser, offset int64) error { +// GetTranslateData provides a reader for key translation logs starting at offset. +func (api *API) GetTranslateData(ctx context.Context, offset int64) (io.ReadCloser, error) { rc, err := api.server.translateFile.Reader(ctx, offset) if err != nil { - return errors.Wrap(err, "read from translate store") + return nil, errors.Wrap(err, "read from translate store") } // Ensure reader is closed when the client disconnects. go func() { <-ctx.Done(); rc.Close() }() - go func() { - defer rc.Close() - defer w.Close() - - buf := make([]byte, translateStoreBufferSize) - - // Copy from reader to client until store or client disconnect. - for { - // Read from store. - n, err := rc.Read(buf) - if err == io.EOF { - return - } else if err != nil { - api.server.logger.Printf("api: translate store read error: %s", err) - return - } else if n == 0 { - continue - } - - // Write to response & flush. - if _, err := w.Write(buf[:n]); err != nil { - api.server.logger.Printf("api: translate store response write error: %s", err) - return - } - } - }() - - return nil + return rc, nil } // State returns the cluster state which is usually "NORMAL", but could be diff --git a/executor.go b/executor.go index 6f0e0ce8b..10026ae2c 100644 --- a/executor.go +++ b/executor.go @@ -101,9 +101,12 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } // Translate query keys to ids, if necessary. - for i := range q.Calls { - if err := e.translateCall(index, idx, q.Calls[i]); err != nil { - return nil, err + // No need to translate a remote call. + if !opt.Remote { + for i := range q.Calls { + if err := e.translateCall(index, idx, q.Calls[i]); err != nil { + return nil, err + } } } @@ -113,10 +116,13 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } // Translate response objects from ids to keys, if necessary. - for i := range results { - results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) - if err != nil { - return nil, err + // No need to translate a remote call. + if !opt.Remote { + for i := range results { + results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) + if err != nil { + return nil, err + } } } return results, nil diff --git a/http/handler.go b/http/handler.go index 8e0fcb709..2ac314d34 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1308,14 +1308,14 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques type defaultClusterMessageResponse struct{} +// translateStoreBufferSize is the buffer size used for streaming data. +const translateStoreBufferSize = 65536 + func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64) - pipeR, pipeW := io.Pipe() - - err := h.api.GetTranslateData(r.Context(), pipeW, offset) - + rdr, err := h.api.GetTranslateData(r.Context(), offset) if err != nil { if errors.Cause(err) == pilosa.ErrNotImplemented { http.Error(w, err.Error(), http.StatusNotImplemented) @@ -1331,7 +1331,28 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) w.Flush() } - io.Copy(w, pipeR) + // Copy from reader to client until store or client disconnect. + buf := make([]byte, translateStoreBufferSize) + for { + // Read from store. + n, err := rdr.Read(buf) + if err == io.EOF { + return + } else if err != nil { + h.logger.Printf("http: translate store read error: %s", err) + return + } else if n == 0 { + continue + } + + // Write to response & flush. + if _, err := w.Write(buf[:n]); err != nil { + h.logger.Printf("http: translate store response write error: %s", err) + return + } else if w, ok := w.(http.Flusher); ok { + w.Flush() + } + } } type queryValidationSpec struct { diff --git a/http/translator.go b/http/translator.go index d094a8c28..80c73bf5e 100644 --- a/http/translator.go +++ b/http/translator.go @@ -16,7 +16,7 @@ import ( // Ensure implementation implements inteface. var _ pilosa.TranslateStore = (*translateStore)(nil) -// translateStore represents an implementation of translateStore that +// translateStore represents an implementation of pilosa.TranslateStore that // communicates over HTTP. This is used with the TranslateHandler. type translateStore struct { URL string diff --git a/mock/translator.go b/mock/translator.go index 186c81894..7a63d8cf0 100644 --- a/mock/translator.go +++ b/mock/translator.go @@ -12,8 +12,8 @@ var _ pilosa.TranslateStore = (*TranslateStore)(nil) type TranslateStore struct { TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error) TranslateColumnToStringFunc func(index string, values uint64) (string, error) - TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error) - TranslateRowToStringFunc func(index, frame string, values uint64) (string, error) + TranslateRowsToUint64Func func(index, field string, values []string) ([]uint64, error) + TranslateRowToStringFunc func(index, field string, values uint64) (string, error) ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error) } @@ -25,12 +25,12 @@ func (s TranslateStore) TranslateColumnToString(index string, values uint64) (st return s.TranslateColumnToStringFunc(index, values) } -func (s TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { - return s.TranslateRowsToUint64Func(index, frame, values) +func (s TranslateStore) TranslateRowsToUint64(index, field string, values []string) ([]uint64, error) { + return s.TranslateRowsToUint64Func(index, field, values) } -func (s TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { - return s.TranslateRowToStringFunc(index, frame, value) +func (s TranslateStore) TranslateRowToString(index, field string, value uint64) (string, error) { + return s.TranslateRowToStringFunc(index, field, value) } func (s TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { diff --git a/server/config.go b/server/config.go index 73f0b289e..39a769ce2 100644 --- a/server/config.go +++ b/server/config.go @@ -74,7 +74,7 @@ type Config struct { // Translation config supports translation store replication. Translation struct { PrimaryURL string `toml:"primary-url"` - } + } `toml:"translation"` AntiEntropy struct { Interval toml.Duration `toml:"interval"` diff --git a/translate.go b/translate.go index 6b3c1856b..09eed00b2 100644 --- a/translate.go +++ b/translate.go @@ -39,8 +39,8 @@ type TranslateStore interface { TranslateColumnsToUint64(index string, values []string) ([]uint64, error) TranslateColumnToString(index string, values uint64) (string, error) - TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) - TranslateRowToString(index, frame string, values uint64) (string, error) + TranslateRowsToUint64(index, field string, values []string) ([]uint64, error) + TranslateRowToString(index, field string, values uint64) (string, error) // Returns a reader from the given offset of the raw data file. // The returned reader must be closed by the caller when done. @@ -64,7 +64,7 @@ type TranslateFile struct { closing chan struct{} cols map[string]*index - rows map[frameKey]*index + rows map[fieldKey]*index Path string mapSize int @@ -82,7 +82,7 @@ func NewTranslateFile() *TranslateFile { writeNotify: make(chan struct{}), closing: make(chan struct{}), cols: make(map[string]*index), - rows: make(map[frameKey]*index), + rows: make(map[fieldKey]*index), mapSize: defaultMapSize, @@ -201,7 +201,7 @@ func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error { idx = s.col(string(entry.Index)) case LogEntryTypeInsertRow: - idx = s.row(string(entry.Index), string(entry.Frame)) + idx = s.row(string(entry.Index), string(entry.Field)) default: return fmt.Errorf("enterprise.TranslateFile.applyEntry(): unknown log entry type: 0x%20x", entry.Type) @@ -318,11 +318,11 @@ func (s *TranslateFile) col(index string) *index { return idx } -func (s *TranslateFile) row(index, frame string) *index { - idx := s.rows[frameKey{index, frame}] +func (s *TranslateFile) row(index, field string) *index { + idx := s.rows[fieldKey{index, field}] if idx == nil { idx = newIndex(s.data) - s.rows[frameKey{index, frame}] = idx + s.rows[fieldKey{index, field}] = idx } return idx } @@ -433,8 +433,8 @@ func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (str return "", nil } -func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { - key := frameKey{index, frame} +func (s *TranslateFile) TranslateRowsToUint64(index, field string, values []string) ([]uint64, error) { + key := fieldKey{index, field} ret := make([]uint64, len(values)) @@ -495,7 +495,7 @@ func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []stri entry := &LogEntry{ Type: LogEntryTypeInsertRow, Index: []byte(index), - Frame: []byte(frame), + Field: []byte(field), IDs: make([]uint64, 0, len(values)), Keys: make([][]byte, 0, len(values)), } @@ -524,9 +524,9 @@ func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []stri return ret, nil } -func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (string, error) { +func (s *TranslateFile) TranslateRowToString(index, field string, id uint64) (string, error) { s.mu.RLock() - if idx := s.rows[frameKey{index, frame}]; idx != nil { + if idx := s.rows[fieldKey{index, field}]; idx != nil { if ret, ok := idx.keyByID(id); ok { s.mu.RUnlock() return string(ret), nil @@ -548,7 +548,7 @@ func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser type LogEntry struct { Type uint8 Index []byte - Frame []byte + Field []byte IDs []uint64 Keys [][]byte @@ -558,12 +558,12 @@ type LogEntry struct { Length uint64 } -// headerSize returns the number of bytes required for size, type, index, frame, & pair count. +// headerSize returns the number of bytes required for size, type, index, field, & pair count. func (e *LogEntry) headerSize() int64 { sz := uVarintSize(e.Length) + // total entry length 1 + // type uVarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data - uVarintSize(uint64(len(e.Frame))) + len(e.Frame) + // Frame length and data + uVarintSize(uint64(len(e.Field))) + len(e.Field) + // Field length and data uVarintSize(uint64(len(e.IDs))) // ID/Key pair count return int64(sz) } @@ -604,14 +604,14 @@ func (e *LogEntry) ReadFrom(r io.Reader) (_ int64, err error) { } } - // Read frame name. + // Read field name. if sz, err := binary.ReadUvarint(br); err != nil { return n64, err } else if sz == 0 { - e.Frame = nil + e.Field = nil } else { - e.Frame = make([]byte, sz) - if _, err := io.ReadFull(r, e.Frame); err != nil { + e.Field = make([]byte, sz) + if _, err := io.ReadFull(r, e.Field); err != nil { return n64, err } } @@ -663,11 +663,11 @@ func (e *LogEntry) WriteTo(w io.Writer) (_ int64, err error) { return 0, err } - // Write frame name. - sz = binary.PutUvarint(b, uint64(len(e.Frame))) + // Write field name. + sz = binary.PutUvarint(b, uint64(len(e.Field))) if _, err := buf.Write(b[:sz]); err != nil { return 0, err - } else if _, err := buf.Write(e.Frame); err != nil { + } else if _, err := buf.Write(e.Field); err != nil { return 0, err } @@ -722,9 +722,9 @@ func validLogEntriesLen(p []byte) (n int) { } } -type frameKey struct { +type fieldKey struct { index string - frame string + field string } const defaultLoadFactor = 90 @@ -910,7 +910,7 @@ type translateFileReader struct { closing chan struct{} } -// newTranslateFileReader returns a new instance of TranslateFileReader. +// newTranslateFileReader returns a new instance of translateFileReader. func newTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *translateFileReader { return &translateFileReader{ ctx: ctx, diff --git a/translate_test.go b/translate_test.go index 693cca759..e41a4dd79 100644 --- a/translate_test.go +++ b/translate_test.go @@ -136,42 +136,42 @@ func TestTranslateFile_TranslateRow(t *testing.T) { defer s.MustClose() // First translation should start id at zero. - if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"foo"}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ids, []uint64{1}) { t.Fatalf("unexpected id: %#v", ids) } // Next translation on the same index should move to one. - if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar"}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ids, []uint64{2}) { t.Fatalf("unexpected id: %#v", ids) } // Translation on a different index restarts at 0. - if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + if ids, err := s.TranslateRowsToUint64("IDX1", "FIELD0", []string{"bar"}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ids, []uint64{1}) { t.Fatalf("unexpected id: %#v", ids) } - // Translation on a different frame restarts at 0. - if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ids, []uint64{1}) { t.Fatalf("unexpected id: %#v", ids) } // Ensure that string values can be looked up by ID. - if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { t.Fatal(err) } else if value != "bar" { t.Fatalf("unexpected value: %s", value) } // Ensure that non-existent values return blank. - if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil { + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 1000); err != nil { t.Fatal(err) } else if value != "" { t.Fatalf("unexpected value: %s", value) @@ -182,22 +182,22 @@ func TestTranslateFile_TranslateRow(t *testing.T) { t.Fatal(err) } - // Translation on a different frame restarts at 0. - if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ids, []uint64{1}) { t.Fatalf("unexpected id: %#v", ids) } // Ensure that string values can be looked up by ID. - if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { t.Fatal(err) } else if value != "bar" { t.Fatalf("unexpected value: %s", value) } // Translate new row and increment sequence. - if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil { + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"baz"}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ids, []uint64{3}) { t.Fatalf("unexpected id: %#v", ids) @@ -215,7 +215,7 @@ func TestTranslateFile_TranslateRow_Large(t *testing.T) { keys[j] = strconv.Itoa(i + j + 1) } - ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys) + ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", keys) if err != nil { t.Fatal(err) } @@ -230,7 +230,7 @@ func TestTranslateFile_TranslateRow_Large(t *testing.T) { // Verify values can be returned. for i := 0; i < 1000000; i++ { exp := strconv.Itoa(i + 1) - if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + if key, err := s.TranslateRowToString("IDX0", "FIELD0", uint64(i+1)); err != nil { t.Fatal(err) } else if key != exp { t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) @@ -243,7 +243,7 @@ func TestTranslateFile_TranslateRow_Large(t *testing.T) { } for i := 0; i < 1000000; i++ { exp := strconv.Itoa(i + 1) - if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + if key, err := s.TranslateRowToString("IDX0", "FIELD0", uint64(i+1)); err != nil { t.Fatal(err) } else if key != exp { t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) @@ -257,7 +257,7 @@ func TestTranslateFile_Reader(t *testing.T) { defer s.MustClose() if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { t.Fatal(err) - } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + } else if _, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar", "baz"}); err != nil { t.Fatal(err) } @@ -290,7 +290,7 @@ func TestTranslateFile_Reader(t *testing.T) { } else if diff := cmp.Diff(entry, pilosa.LogEntry{ Type: pilosa.LogEntryTypeInsertRow, Index: []byte("IDX0"), - Frame: []byte("FRAME0"), + Field: []byte("FIELD0"), IDs: []uint64{1, 2}, Keys: [][]byte{[]byte("bar"), []byte("baz")}, Length: 24, @@ -329,7 +329,7 @@ func TestTranslateFile_Reader(t *testing.T) { defer s.MustClose() if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { t.Fatal(err) - } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + } else if _, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar", "baz"}); err != nil { t.Fatal(err) } @@ -348,7 +348,7 @@ func TestTranslateFile_Reader(t *testing.T) { } else if diff := cmp.Diff(entry, pilosa.LogEntry{ Type: pilosa.LogEntryTypeInsertRow, Index: []byte("IDX0"), - Frame: []byte("FRAME0"), + Field: []byte("FIELD0"), IDs: []uint64{1, 2}, Keys: [][]byte{[]byte("bar"), []byte("baz")}, Length: 24, @@ -392,7 +392,7 @@ func TestTranslateFile_PrimaryTranslateStore(t *testing.T) { // Write to the primary. if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { t.Fatal(err) - } else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + } else if _, err := primary.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar", "baz"}); err != nil { t.Fatal(err) } @@ -405,13 +405,13 @@ func TestTranslateFile_PrimaryTranslateStore(t *testing.T) { return fmt.Errorf("unexpected column 1 value: %s", value) } - if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil { + if value, err := replica.TranslateRowToString("IDX0", "FIELD0", 1); err != nil { return err } else if value != "bar" { return fmt.Errorf("unexpected row 1 value: %s", value) } - if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + if value, err := replica.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { return err } else if value != "baz" { return fmt.Errorf("unexpected row 2 value: %s", value) From ac8f8aa2e47465e85744c00fe0dc92dc7f0cb1a7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 30 Jul 2018 11:44:29 -0500 Subject: [PATCH 09/17] remove incorrect mock implementation from tranlate test --- http/translator_test.go | 87 ++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/http/translator_test.go b/http/translator_test.go index 04531dab5..05a4da2e6 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -2,9 +2,9 @@ package http_test import ( "context" + "fmt" "io" "io/ioutil" - gohttp "net/http" "testing" "time" @@ -30,74 +30,46 @@ func TestTranslateStore_Reader(t *testing.T) { // Ensure client can connect and stream the translate store data. t.Run("OK", func(t *testing.T) { t.Run("ServerDisconnect", func(t *testing.T) { - var mrc mock.ReadCloser - var readN int - mrc.ReadFunc = func(p []byte) (int, error) { - readN++ - switch readN { - case 1: - copy(p, []byte("foo")) - return 3, nil - case 2: - copy(p, []byte("barbaz")) - return 6, nil - case 3: - return 0, io.EOF - default: - t.Fatal("unexpected read") - return 0, nil - } - } - closeInvoked := make(chan struct{}) - mrc.CloseFunc = func() error { - close(closeInvoked) - return nil + primary := test.MustRunCluster(t, 1)[0] + + hldr := test.Holder{Holder: primary.Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) } - // Setup handler on test server. - var translateStore mock.TranslateStore - - translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { - // Check context to make sure this is the call we are looking for. - // (Something else calls ReaderFunc on server startup) - if ctx.Value(gohttp.ServerContextKey) != nil { - if off != 100 { - t.Fatalf("unexpected off: %d", off) - } - return &mrc, nil - } - return newMockReadCloser(), nil + // Set data on the primary node. + if _, err := primary.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%s, f=%d)\n", `"foo"`, 10) + + fmt.Sprintf("Set(%s, f=%d)\n", `"bar"`, 10) + + fmt.Sprintf("Set(%s, f=%d)\n", `"baz"`, 10), + }); err != nil { + t.Fatal(err) } - opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] - - defer main.Close() - // Connect to server and stream all available data. - store := http.NewTranslateStore(main.URL()) + store := http.NewTranslateStore(primary.URL()) + + rc, err := store.Reader(context.Background(), 11) // offset=11 skips the first entry: \n\x01\x01i\x00\x01\x01\x03foo + + // Close the primary to disconnect reader. + primary.Close() - rc, err := store.Reader(context.Background(), 100) if err != nil { t.Fatal(err) } else if data, err := ioutil.ReadAll(rc); err != nil { t.Fatal(err) - } else if string(data) != `foobarbaz` { + } else if string(data) != "\n\x01\x01i\x00\x01\x02\x03bar\n\x01\x01i\x00\x01\x03\x03baz" { t.Fatalf("unexpected data: %q", data) } else if err := rc.Close(); err != nil { t.Fatal(err) } - - select { - case <-time.NewTimer(time.Millisecond * 100).C: - t.Fatal("expected server close") - case <-closeInvoked: - return - } }) // Ensure server closes store reader if client disconnects. t.Run("ClientDisconnect", func(t *testing.T) { + t.Skip() // can't mock server from http package // Setup mock so that Read() hangs. done := make(chan struct{}) @@ -121,14 +93,14 @@ func TestTranslateStore_Reader(t *testing.T) { } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] + primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] - defer main.Close() + defer primary.Close() defer close(done) // Connect to server and begin streaming. ctx, cancel := context.WithCancel(context.Background()) - store := http.NewTranslateStore(main.URL()) + store := http.NewTranslateStore(primary.URL()) if _, err := store.Reader(ctx, 0); err != nil { t.Fatal(err) } @@ -146,16 +118,17 @@ func TestTranslateStore_Reader(t *testing.T) { // Ensure client is notified if the server doesn't support streaming replication. t.Run("ErrNotImplemented", func(t *testing.T) { + t.Skip() // can't mock server from http package var translateStore mock.TranslateStore translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { return nil, pilosa.ErrNotImplemented } opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore)) - main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] - defer main.Close() + primary := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0] + defer primary.Close() - _, err := http.NewTranslateStore(main.URL()).Reader(context.Background(), 0) + _, err := http.NewTranslateStore(primary.URL()).Reader(context.Background(), 0) if err != pilosa.ErrNotImplemented { t.Fatalf("unexpected error: %s", err) } From e79cd4b199c62dfc1952e0cd77b2595c8a545160 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Jul 2018 11:40:30 -0500 Subject: [PATCH 10/17] Add JSON parsing to translator test to verify keys --- server/handler_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/server/handler_test.go b/server/handler_test.go index 2a319c41b..954523e04 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -681,6 +681,25 @@ func TestClusterTranslator(t *testing.T) { if result0 != result1 { t.Fatalf("`%s` != `%s`", result0, result1) } + + for _, i := range []string{result0, result1} { + var resp map[string]interface{} + err := json.Unmarshal([]byte(i), &resp) + if err != nil { + t.Fatalf("json unmarshal error: %s", err) + } + if results, ok := resp["results"].([]interface{}); ok { + if result, ok := results[0].(map[string]interface{}); ok { + if keys, ok := result["keys"].([]interface{}); ok { + if key, ok := keys[0].(string); ok { + if key != "foo" { + t.Fatalf("Key is %s but should be 'foo'", key) + } + } + } + } + } + } } func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { From 0816ea8ecb48d2b1b988026f135e6fe28b7bcc09 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Jul 2018 12:11:21 -0500 Subject: [PATCH 11/17] Add sleep to test to wait for key replication --- server/handler_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/handler_test.go b/server/handler_test.go index 954523e04..5617c8ffa 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -24,6 +24,7 @@ import ( "reflect" "strings" "testing" + "time" gohttp "net/http" @@ -675,6 +676,9 @@ func TestClusterTranslator(t *testing.T) { test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")") + // wait for key to replicate to second node + time.Sleep(100 * time.Millisecond) + result0 := test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body result1 := test.MustDo("POST", cluster[1].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body From be60a91a58f9fbabe2b456774cd32c952f99a6e0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Jul 2018 12:19:09 -0500 Subject: [PATCH 12/17] Clarify error string for cases when reading from non-primary translate store when given non-existent key --- translate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translate.go b/translate.go index 09eed00b2..00f11723e 100644 --- a/translate.go +++ b/translate.go @@ -31,7 +31,7 @@ var ( ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") ErrReplicationNotSupported = errors.New("pilosa: replication not supported") - ErrTranslateStoreReadOnly = errors.New("pilosa: operation not supported, translate store read only") + ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only") ) // TranslateStore is the storage for translation string-to-uint64 values. From 39a82091f2feb8b8cbbf3186371b52199401dd90 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Jul 2018 14:11:24 -0500 Subject: [PATCH 13/17] Add sleep in tests to ensure writes make it to translate store --- http/translator_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/translator_test.go b/http/translator_test.go index 05a4da2e6..bc882c23f 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -48,6 +48,9 @@ func TestTranslateStore_Reader(t *testing.T) { t.Fatal(err) } + // Wait to ensure writes make it to translate store + time.Sleep(100 * time.Millisecond) + // Connect to server and stream all available data. store := http.NewTranslateStore(primary.URL()) From ba3bda6ac25403fac902e5f5cdfd7faf3db2637a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Jul 2018 14:16:17 -0500 Subject: [PATCH 14/17] Use string prefix instead of equality so json error message will pass on all Go versions --- http/handler_internal_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index 0d00d99c3..b0852a79d 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/json" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" @@ -68,7 +69,7 @@ func TestPostFieldRequestUnmarshalJSON(t *testing.T) { err string }{ {json: `{"options": {}}`, expected: postFieldRequest{}}, - {json: `{"options": 4}`, err: "json: cannot unmarshal number into Go struct field postFieldRequest.options of type http.fieldOptions"}, + {json: `{"options": 4}`, err: "json: cannot unmarshal number"}, {json: `{"option": {}}`, err: `json: unknown field "option"`}, {json: `{"options": {"badKey": "test"}}`, err: `json: unknown field "badKey"`}, {json: `{"options": {"inverseEnabled": true}}`, err: `json: unknown field "inverseEnabled"`}, @@ -81,7 +82,7 @@ func TestPostFieldRequestUnmarshalJSON(t *testing.T) { dec.DisallowUnknownFields() err := dec.Decode(actual) if err != nil { - if test.err == "" || test.err != err.Error() { + if test.err == "" || !strings.HasPrefix(err.Error(), test.err) { t.Errorf("test %d: expected error: %v, but got result: %v", i, test.err, err) } } From 2dd655f0e93e7a1da471791988ed8f6fda0b4313 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 31 Jul 2018 15:59:21 -0500 Subject: [PATCH 15/17] move sleep to be after replica sync --- http/translator_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/http/translator_test.go b/http/translator_test.go index bc882c23f..458ea895c 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -48,12 +48,12 @@ func TestTranslateStore_Reader(t *testing.T) { t.Fatal(err) } - // Wait to ensure writes make it to translate store - time.Sleep(100 * time.Millisecond) - // Connect to server and stream all available data. store := http.NewTranslateStore(primary.URL()) + // Wait to ensure writes make it to translate store + time.Sleep(100 * time.Millisecond) + rc, err := store.Reader(context.Background(), 11) // offset=11 skips the first entry: \n\x01\x01i\x00\x01\x01\x03foo // Close the primary to disconnect reader. From c44bb651637e09d813196fccbcac4a03b85748b3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Jul 2018 20:38:18 -0500 Subject: [PATCH 16/17] Add more time to sleep in translator tests due to CI failures --- http/translator_test.go | 2 +- server/handler_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/http/translator_test.go b/http/translator_test.go index 458ea895c..660e2e0aa 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -52,7 +52,7 @@ func TestTranslateStore_Reader(t *testing.T) { store := http.NewTranslateStore(primary.URL()) // Wait to ensure writes make it to translate store - time.Sleep(100 * time.Millisecond) + time.Sleep(500 * time.Millisecond) rc, err := store.Reader(context.Background(), 11) // offset=11 skips the first entry: \n\x01\x01i\x00\x01\x01\x03foo diff --git a/server/handler_test.go b/server/handler_test.go index 5617c8ffa..487756c45 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -677,7 +677,7 @@ func TestClusterTranslator(t *testing.T) { test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")") // wait for key to replicate to second node - time.Sleep(100 * time.Millisecond) + time.Sleep(500 * time.Millisecond) result0 := test.MustDo("POST", cluster[0].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body result1 := test.MustDo("POST", cluster[1].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body From 495b4259fd03ed020347fd0123af58b07703c690 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 1 Aug 2018 10:57:56 -0500 Subject: [PATCH 17/17] Skip flawed translator test --- http/translator_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/http/translator_test.go b/http/translator_test.go index 660e2e0aa..98f5b1403 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -30,6 +30,10 @@ func TestTranslateStore_Reader(t *testing.T) { // Ensure client can connect and stream the translate store data. t.Run("OK", func(t *testing.T) { t.Run("ServerDisconnect", func(t *testing.T) { + // This test is currently flawed, breaking intermittently with message: + // "translator_test.go:65: unexpected EOF" + t.Skip() + primary := test.MustRunCluster(t, 1)[0] hldr := test.Holder{Holder: primary.Server.Holder()}