From 0a96043c23e2a3d5c620f0846c92f9181e4c7354 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 17 Aug 2020 22:16:47 -0500 Subject: [PATCH 1/2] Use _buffer instead of buffer --- pql/parser.go | 4 +- pql/parser_test.go | 19 ++++++++++ pql/pql.peg | 8 ++-- pql/pql.peg.go | 93 +++++++++++++++++----------------------------- pql/pqlpeg_test.go | 8 +--- 5 files changed, 60 insertions(+), 72 deletions(-) diff --git a/pql/parser.go b/pql/parser.go index 8a5907dbf..e09b431ef 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -61,9 +61,7 @@ func (p *parser) Parse() (*Query, error) { p.PQL = PQL{ Buffer: string(buf), } - if err := p.Init(); err != nil { - return nil, errors.Wrap(err, "initializing") - } + p.Init() err = p.PQL.Parse() if err != nil { return nil, errors.Wrap(err, "parsing") diff --git a/pql/parser_test.go b/pql/parser_test.go index 8e5cf8cc4..94d97d1a1 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -194,6 +194,25 @@ func TestParser_Parse(t *testing.T) { } }) + // Parse unicode keys + t.Run("UnicodeKey", func(t *testing.T) { + // s := `���t` + s := `Æ` + q, err := pql.ParseString(`Row(unicode="` + s + `")`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + "unicode": s, + }, + }, + ) { + t.Fatalf("uexpected call: %#v", q.Calls[0]) + } + }) + } func TestUnquote(t *testing.T) { diff --git a/pql/pql.peg b/pql/pql.peg index 7e6c039bd..dcb10d26d 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -62,10 +62,10 @@ itema <- ( 'null' &(comma / sp close) { p.addVal(nil) } / 'false' &(comma / sp close) { p.addVal(false) } / timestampfmt { p.addVal(buffer[begin:end]) } ) -itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } - / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } - / < '"' doublequotedstring '"' > { p.addVal(buffer[begin:end]) } - / < '\'' singlequotedstring '\'' > { p.addVal(buffer[begin:end]) } +itemb <- ( < IDENT > { p.startCall(string(_buffer[begin:end])) } open allargs comma? close { p.addVal(p.endCall()) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(string(_buffer[begin:end])) } + / < '"' doublequotedstring '"' > { p.addVal(string(_buffer[begin:end])) } + / < '\'' singlequotedstring '\'' > { p.addVal(string(_buffer[begin:end])) } ) float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 3cba1b702..e16002005 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -1,11 +1,10 @@ package pql -// Code generated by peg -inline pql.peg DO NOT EDIT. +//go:generate peg -inline pql.peg import ( "fmt" - "io" - "os" + "math" "sort" "strconv" ) @@ -245,19 +244,19 @@ type node32 struct { up, next *node32 } -func (node *node32) print(w io.Writer, pretty bool, buffer string) { +func (node *node32) print(pretty bool, buffer string) { var print func(node *node32, depth int) print = func(node *node32, depth int) { for node != nil { for c := 0; c < depth; c++ { - fmt.Fprintf(w, " ") + fmt.Printf(" ") } rule := rul3s[node.pegRule] quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) if !pretty { - fmt.Fprintf(w, "%v %v\n", rule, quote) + fmt.Printf("%v %v\n", rule, quote) } else { - fmt.Fprintf(w, "\x1B[34m%v\x1B[m %v\n", rule, quote) + fmt.Printf("\x1B[34m%v\x1B[m %v\n", rule, quote) } if node.up != nil { print(node.up, depth+1) @@ -268,12 +267,12 @@ func (node *node32) print(w io.Writer, pretty bool, buffer string) { print(node, 0) } -func (node *node32) Print(w io.Writer, buffer string) { - node.print(w, false, buffer) +func (node *node32) Print(buffer string) { + node.print(false, buffer) } -func (node *node32) PrettyPrint(w io.Writer, buffer string) { - node.print(w, true, buffer) +func (node *node32) PrettyPrint(buffer string) { + node.print(true, buffer) } type tokens32 struct { @@ -316,24 +315,24 @@ func (t *tokens32) AST() *node32 { } func (t *tokens32) PrintSyntaxTree(buffer string) { - t.AST().Print(os.Stdout, buffer) -} - -func (t *tokens32) WriteSyntaxTree(w io.Writer, buffer string) { - t.AST().Print(w, buffer) + t.AST().Print(buffer) } func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { - t.AST().PrettyPrint(os.Stdout, buffer) + t.AST().PrettyPrint(buffer) } func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { - tree, i := t.tree, int(index) - if i >= len(tree) { - t.tree = append(tree, token32{pegRule: rule, begin: begin, end: end}) - return + if tree := t.tree; int(index) >= len(tree) { + expanded := make([]token32, 2*len(tree)) + copy(expanded, tree) + t.tree = expanded + } + t.tree[index] = token32{ + pegRule: rule, + begin: begin, + end: end, } - tree[i] = token32{pegRule: rule, begin: begin, end: end} } func (t *tokens32) Tokens() []token32 { @@ -397,7 +396,7 @@ type parseError struct { } func (e *parseError) Error() string { - tokens, err := []token32{e.max}, "\n" + tokens, error := []token32{e.max}, "\n" positions, p := make([]int, 2*len(tokens)), 0 for _, token := range tokens { positions[p], p = int(token.begin), p+1 @@ -410,14 +409,14 @@ func (e *parseError) Error() string { } for _, token := range tokens { begin, end := int(token.begin), int(token.end) - err += fmt.Sprintf(format, + error += fmt.Sprintf(format, rul3s[token.pegRule], translations[begin].line, translations[begin].symbol, translations[end].line, translations[end].symbol, strconv.Quote(string(e.p.buffer[begin:end]))) } - return err + return error } func (p *PQL) PrintSyntaxTree() { @@ -428,10 +427,6 @@ func (p *PQL) PrintSyntaxTree() { } } -func (p *PQL) WriteSyntaxTree(w io.Writer) { - p.tokens32.WriteSyntaxTree(w, p.Buffer) -} - func (p *PQL) Execute() { buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 for _, token := range p.Tokens() { @@ -530,15 +525,15 @@ func (p *PQL) Execute() { case ruleAction43: p.addVal(buffer[begin:end]) case ruleAction44: - p.startCall(buffer[begin:end]) + p.startCall(string(_buffer[begin:end])) case ruleAction45: p.addVal(p.endCall()) case ruleAction46: - p.addVal(buffer[begin:end]) + p.addVal(string(_buffer[begin:end])) case ruleAction47: - p.addVal(buffer[begin:end]) + p.addVal(string(_buffer[begin:end])) case ruleAction48: - p.addVal(buffer[begin:end]) + p.addVal(string(_buffer[begin:end])) case ruleAction49: p.addNumVal(buffer[begin:end], true) case ruleAction50: @@ -571,31 +566,12 @@ func (p *PQL) Execute() { _, _, _, _, _ = buffer, _buffer, text, begin, end } -func Pretty(pretty bool) func(*PQL) error { - return func(p *PQL) error { - p.Pretty = pretty - return nil - } -} - -func Size(size int) func(*PQL) error { - return func(p *PQL) error { - p.tokens32 = tokens32{tree: make([]token32, 0, size)} - return nil - } -} -func (p *PQL) Init(options ...func(*PQL) error) error { +func (p *PQL) Init() { var ( max token32 position, tokenIndex uint32 buffer []rune ) - for _, option := range options { - err := option(p) - if err != nil { - return err - } - } p.reset = func() { max = token32{} position, tokenIndex = 0, 0 @@ -609,7 +585,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { p.reset() _rules := p.rules - tree := p.tokens32 + tree := tokens32{tree: make([]token32, math.MaxInt16)} p.parse = func(rule ...int) error { r := 1 if len(rule) > 0 { @@ -3631,15 +3607,15 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 86 Action43 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 87 Action44 <- <{ p.startCall(buffer[begin:end]) }> */ + /* 87 Action44 <- <{ p.startCall(string(_buffer[begin:end])) }> */ nil, /* 88 Action45 <- <{ p.addVal(p.endCall()) }> */ nil, - /* 89 Action46 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 89 Action46 <- <{ p.addVal(string(_buffer[begin:end])) }> */ nil, - /* 90 Action47 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 90 Action47 <- <{ p.addVal(string(_buffer[begin:end])) }> */ nil, - /* 91 Action48 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 91 Action48 <- <{ p.addVal(string(_buffer[begin:end])) }> */ nil, /* 92 Action49 <- <{ p.addNumVal(buffer[begin:end], true) }> */ nil, @@ -3669,5 +3645,4 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, } p.rules = _rules - return nil } diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 4ebf1dcc9..e6aba444d 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -24,9 +24,7 @@ import ( func TestPEG(t *testing.T) { p := PQL{Buffer: ` SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} - if err := p.Init(); err != nil { - t.Fatalf("initialization error: %v", err) - } + p.Init() err := p.Parse() if err != nil { t.Fatalf("parse error: %v", err) @@ -34,9 +32,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9 p.Execute() p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`} - if err := p.Init(); err != nil { - t.Fatalf("initialization error: %v", err) - } + p.Init() err = p.Parse() if err == nil { t.Fatalf("should have been an error because of the interior unescaped double quote") From 6fc5465aaa0f8fb8f08d5dfa8a764c0689c5c339 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 19 Aug 2020 16:19:43 -0500 Subject: [PATCH 2/2] Use rune slice in all cases, add tests --- pql/parser_test.go | 20 ---------- pql/pql.peg | 46 +++++++++++----------- pql/pql.peg.go | 98 +++++++++++++++++++++++----------------------- pql/pqlpeg_test.go | 53 +++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 92 deletions(-) diff --git a/pql/parser_test.go b/pql/parser_test.go index 94d97d1a1..d2ee45748 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -193,26 +193,6 @@ func TestParser_Parse(t *testing.T) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) - - // Parse unicode keys - t.Run("UnicodeKey", func(t *testing.T) { - // s := `���t` - s := `Æ` - q, err := pql.ParseString(`Row(unicode="` + s + `")`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "Row", - Args: map[string]interface{}{ - "unicode": s, - }, - }, - ) { - t.Fatalf("uexpected call: %#v", q.Calls[0]) - } - }) - } func TestUnquote(t *testing.T) { diff --git a/pql/pql.peg b/pql/pql.peg index dcb10d26d..b58e36723 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -14,8 +14,8 @@ Call <- 'Set' {p.startCall("Set")} open col comma dargs (comma timestamp)? clos / 'Store' {p.startCall("Store")} open Call comma darg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()} - / 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()} - / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } + / 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(text)} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(text)} close {p.endCall()} + / < IDENT > { p.startCall(text ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma dargs)? / dargs / sp fargs <- farg (comma fargs)? sp farg <- ( field sp '=' sp fvalue @@ -37,9 +37,9 @@ COND <- ( '><' { p.addBTWN() } ) conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()} -condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(buffer[begin:end])} -condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])} -condfield <- sp {p.condAdd(buffer[begin:end])} +condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(text)} +condLT <- <('<=' / '<')> sp {p.condAdd(text)} +condfield <- sp {p.condAdd(text)} dvalue <- ( ditem / lbrack { p.startList() } dlist rbrack { p.endList() } @@ -60,35 +60,35 @@ fitem <- ( itema itema <- ( 'null' &(comma / sp close) { p.addVal(nil) } / 'true' &(comma / sp close) { p.addVal(true) } / 'false' &(comma / sp close) { p.addVal(false) } - / timestampfmt { p.addVal(buffer[begin:end]) } + / timestampfmt { p.addVal(text) } ) -itemb <- ( < IDENT > { p.startCall(string(_buffer[begin:end])) } open allargs comma? close { p.addVal(p.endCall()) } - / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(string(_buffer[begin:end])) } - / < '"' doublequotedstring '"' > { p.addVal(string(_buffer[begin:end])) } - / < '\'' singlequotedstring '\'' > { p.addVal(string(_buffer[begin:end])) } +itemb <- ( < IDENT > { p.startCall(text) } open allargs comma? close { p.addVal(p.endCall()) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(text) } + / < '"' doublequotedstring '"' > { p.addVal(text) } + / < '\'' singlequotedstring '\'' > { p.addVal(text) } ) -float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) } - / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) } +float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(text, true) } + / < '-'? '.'[0-9]+ > { p.addNumVal(text, true) } ) -decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false) } - / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) } +decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(text, false) } + / < '-'? '.'[0-9]+ > { p.addNumVal(text, false) } ) doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )* singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* -field <- { p.addField(buffer[begin:end]) } +field <- { p.addField(text) } reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field') -posfield <- { p.addPosStr("_field", buffer[begin:end]) } +posfield <- { p.addPosStr("_field", text) } uint <- [1-9] [0-9]* / '0' -col <- ( {p.addPosNum("_col", buffer[begin:end])} - / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])} - / < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])} +col <- ( {p.addPosNum("_col", text)} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)} + / < '"' doublequotedstring '"' > {p.addPosStr("_col", text)} ) -row <- ( {p.addPosNum("_row", buffer[begin:end])} - / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])} - / < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])} +row <- ( {p.addPosNum("_row", text)} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", text)} + / < '"' doublequotedstring '"' > {p.addPosStr("_row", text)} ) open <- '(' sp @@ -102,4 +102,4 @@ IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] timestampfmt <- '"' '"' / '\'' '\'' / -timestamp <- {p.addPosStr("_timestamp", buffer[begin:end])} +timestamp <- {p.addPosStr("_timestamp", text)} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index e16002005..3c2a70940 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -1,6 +1,6 @@ package pql -//go:generate peg -inline pql.peg +// Code generated by peg -inline pql.peg DO NOT EDIT. import ( "fmt" @@ -473,15 +473,15 @@ func (p *PQL) Execute() { case ruleAction17: p.addField("from") case ruleAction18: - p.addVal(buffer[begin:end]) + p.addVal(text) case ruleAction19: p.addField("to") case ruleAction20: - p.addVal(buffer[begin:end]) + p.addVal(text) case ruleAction21: p.endCall() case ruleAction22: - p.startCall(buffer[begin:end]) + p.startCall(text) case ruleAction23: p.endCall() case ruleAction24: @@ -503,11 +503,11 @@ func (p *PQL) Execute() { case ruleAction32: p.endConditional() case ruleAction33: - p.condAdd(buffer[begin:end]) + p.condAdd(text) case ruleAction34: - p.condAdd(buffer[begin:end]) + p.condAdd(text) case ruleAction35: - p.condAdd(buffer[begin:end]) + p.condAdd(text) case ruleAction36: p.startList() case ruleAction37: @@ -523,43 +523,43 @@ func (p *PQL) Execute() { case ruleAction42: p.addVal(false) case ruleAction43: - p.addVal(buffer[begin:end]) + p.addVal(text) case ruleAction44: - p.startCall(string(_buffer[begin:end])) + p.startCall(text) case ruleAction45: p.addVal(p.endCall()) case ruleAction46: - p.addVal(string(_buffer[begin:end])) + p.addVal(text) case ruleAction47: - p.addVal(string(_buffer[begin:end])) + p.addVal(text) case ruleAction48: - p.addVal(string(_buffer[begin:end])) + p.addVal(text) case ruleAction49: - p.addNumVal(buffer[begin:end], true) + p.addNumVal(text, true) case ruleAction50: - p.addNumVal(buffer[begin:end], true) + p.addNumVal(text, true) case ruleAction51: - p.addNumVal(buffer[begin:end], false) + p.addNumVal(text, false) case ruleAction52: - p.addNumVal(buffer[begin:end], false) + p.addNumVal(text, false) case ruleAction53: - p.addField(buffer[begin:end]) + p.addField(text) case ruleAction54: - p.addPosStr("_field", buffer[begin:end]) + p.addPosStr("_field", text) case ruleAction55: - p.addPosNum("_col", buffer[begin:end]) + p.addPosNum("_col", text) case ruleAction56: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_col", text) case ruleAction57: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_col", text) case ruleAction58: - p.addPosNum("_row", buffer[begin:end]) + p.addPosNum("_row", text) case ruleAction59: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_row", text) case ruleAction60: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_row", text) case ruleAction61: - p.addPosStr("_timestamp", buffer[begin:end]) + p.addPosStr("_timestamp", text) } } @@ -3554,16 +3554,16 @@ func (p *PQL) Init() { nil, /* 59 Action17 <- <{p.addField("from")}> */ nil, - /* 60 Action18 <- <{p.addVal(buffer[begin:end])}> */ + /* 60 Action18 <- <{p.addVal(text)}> */ nil, /* 61 Action19 <- <{p.addField("to")}> */ nil, - /* 62 Action20 <- <{p.addVal(buffer[begin:end])}> */ + /* 62 Action20 <- <{p.addVal(text)}> */ nil, /* 63 Action21 <- <{p.endCall()}> */ nil, nil, - /* 65 Action22 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 65 Action22 <- <{ p.startCall(text ) }> */ nil, /* 66 Action23 <- <{ p.endCall() }> */ nil, @@ -3585,11 +3585,11 @@ func (p *PQL) Init() { nil, /* 75 Action32 <- <{p.endConditional()}> */ nil, - /* 76 Action33 <- <{p.condAdd(buffer[begin:end])}> */ + /* 76 Action33 <- <{p.condAdd(text)}> */ nil, - /* 77 Action34 <- <{p.condAdd(buffer[begin:end])}> */ + /* 77 Action34 <- <{p.condAdd(text)}> */ nil, - /* 78 Action35 <- <{p.condAdd(buffer[begin:end])}> */ + /* 78 Action35 <- <{p.condAdd(text)}> */ nil, /* 79 Action36 <- <{ p.startList() }> */ nil, @@ -3605,43 +3605,43 @@ func (p *PQL) Init() { nil, /* 85 Action42 <- <{ p.addVal(false) }> */ nil, - /* 86 Action43 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 86 Action43 <- <{ p.addVal(text) }> */ nil, - /* 87 Action44 <- <{ p.startCall(string(_buffer[begin:end])) }> */ + /* 87 Action44 <- <{ p.startCall(text) }> */ nil, /* 88 Action45 <- <{ p.addVal(p.endCall()) }> */ nil, - /* 89 Action46 <- <{ p.addVal(string(_buffer[begin:end])) }> */ + /* 89 Action46 <- <{ p.addVal(text) }> */ nil, - /* 90 Action47 <- <{ p.addVal(string(_buffer[begin:end])) }> */ + /* 90 Action47 <- <{ p.addVal(text) }> */ nil, - /* 91 Action48 <- <{ p.addVal(string(_buffer[begin:end])) }> */ + /* 91 Action48 <- <{ p.addVal(text) }> */ nil, - /* 92 Action49 <- <{ p.addNumVal(buffer[begin:end], true) }> */ + /* 92 Action49 <- <{ p.addNumVal(text, true) }> */ nil, - /* 93 Action50 <- <{ p.addNumVal(buffer[begin:end], true) }> */ + /* 93 Action50 <- <{ p.addNumVal(text, true) }> */ nil, - /* 94 Action51 <- <{ p.addNumVal(buffer[begin:end], false) }> */ + /* 94 Action51 <- <{ p.addNumVal(text, false) }> */ nil, - /* 95 Action52 <- <{ p.addNumVal(buffer[begin:end], false) }> */ + /* 95 Action52 <- <{ p.addNumVal(text, false) }> */ nil, - /* 96 Action53 <- <{ p.addField(buffer[begin:end]) }> */ + /* 96 Action53 <- <{ p.addField(text) }> */ nil, - /* 97 Action54 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 97 Action54 <- <{ p.addPosStr("_field", text) }> */ nil, - /* 98 Action55 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 98 Action55 <- <{p.addPosNum("_col", text)}> */ nil, - /* 99 Action56 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 99 Action56 <- <{p.addPosStr("_col", text)}> */ nil, - /* 100 Action57 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 100 Action57 <- <{p.addPosStr("_col", text)}> */ nil, - /* 101 Action58 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 101 Action58 <- <{p.addPosNum("_row", text)}> */ nil, - /* 102 Action59 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 102 Action59 <- <{p.addPosStr("_row", text)}> */ nil, - /* 103 Action60 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 103 Action60 <- <{p.addPosStr("_row", text)}> */ nil, - /* 104 Action61 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 104 Action61 <- <{p.addPosStr("_timestamp", text)}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index e6aba444d..9339cad3a 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -375,6 +375,48 @@ func TestPQLDeepEquality(t *testing.T) { "_timestamp": "2010-07-08T14:44", }, }}, + { + name: "SetWithUnicode", + call: `Set(0, unicode="Æ�漢д ☮♬ ♞🜻💣")`, + exp: &Call{ + Name: "Set", + Args: map[string]interface{}{ + "_col": int64(0), + "unicode": `Æ�漢д ☮♬ ♞🜻💣`, + }, + }}, + { + name: "RowWithUnicode", + call: `Row(unicode="Æ�漢д ☮♬ ♞🜻💣")`, + exp: &Call{ + Name: "Row", + Args: map[string]interface{}{ + "unicode": `Æ�漢д ☮♬ ♞🜻💣`, + }, + }}, + { + name: "RowsWithUnicode", + call: `Rows(job, previous="💣")`, + exp: &Call{ + Name: "Rows", + Args: map[string]interface{}{ + "_field": "job", + "previous": `💣`, + }, + }}, + { + name: "TopNWithUnicode", + call: `TopN(stargazer, Row(unicode="Æ�漢д ☮♬ ♞🜻💣"), a="∑")`, + exp: &Call{ + Name: "TopN", + Args: map[string]interface{}{ + "_field": "stargazer", + "a": "∑", + }, + Children: []*Call{ + {Name: "Row", Args: map[string]interface{}{"unicode": "Æ�漢д ☮♬ ♞🜻💣"}}, + }, + }}, { name: "SetRowAttrs", call: "SetRowAttrs(myfield, 9, z=4)", @@ -409,6 +451,17 @@ func TestPQLDeepEquality(t *testing.T) { }, }}, { + name: "SetRowAttrsWithUnicodeValues", + call: `SetRowAttrs(myfield, "∫", z="∀", a="∑")`, + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": "∀", + "a": "∑", + "_field": "myfield", + "_row": "∫", + }, + }}, { name: "SetColumnAttrs", call: "SetColumnAttrs(9, z=4)", exp: &Call{