diff --git a/pql/parser.go b/pql/parser.go index 8a5907dbf..e09b431ef 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -61,9 +61,7 @@ func (p *parser) Parse() (*Query, error) { p.PQL = PQL{ Buffer: string(buf), } - if err := p.Init(); err != nil { - return nil, errors.Wrap(err, "initializing") - } + p.Init() err = p.PQL.Parse() if err != nil { return nil, errors.Wrap(err, "parsing") diff --git a/pql/parser_test.go b/pql/parser_test.go index 8e5cf8cc4..94d97d1a1 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -194,6 +194,25 @@ func TestParser_Parse(t *testing.T) { } }) + // Parse unicode keys + t.Run("UnicodeKey", func(t *testing.T) { + // s := `���t` + s := `Æ` + q, err := pql.ParseString(`Row(unicode="` + s + `")`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + "unicode": s, + }, + }, + ) { + t.Fatalf("uexpected call: %#v", q.Calls[0]) + } + }) + } func TestUnquote(t *testing.T) { diff --git a/pql/pql.peg b/pql/pql.peg index 7e6c039bd..dcb10d26d 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -62,10 +62,10 @@ itema <- ( 'null' &(comma / sp close) { p.addVal(nil) } / 'false' &(comma / sp close) { p.addVal(false) } / timestampfmt { p.addVal(buffer[begin:end]) } ) -itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } - / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } - / < '"' doublequotedstring '"' > { p.addVal(buffer[begin:end]) } - / < '\'' singlequotedstring '\'' > { p.addVal(buffer[begin:end]) } +itemb <- ( < IDENT > { p.startCall(string(_buffer[begin:end])) } open allargs comma? close { p.addVal(p.endCall()) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(string(_buffer[begin:end])) } + / < '"' doublequotedstring '"' > { p.addVal(string(_buffer[begin:end])) } + / < '\'' singlequotedstring '\'' > { p.addVal(string(_buffer[begin:end])) } ) float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 3cba1b702..e16002005 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -1,11 +1,10 @@ package pql -// Code generated by peg -inline pql.peg DO NOT EDIT. +//go:generate peg -inline pql.peg import ( "fmt" - "io" - "os" + "math" "sort" "strconv" ) @@ -245,19 +244,19 @@ type node32 struct { up, next *node32 } -func (node *node32) print(w io.Writer, pretty bool, buffer string) { +func (node *node32) print(pretty bool, buffer string) { var print func(node *node32, depth int) print = func(node *node32, depth int) { for node != nil { for c := 0; c < depth; c++ { - fmt.Fprintf(w, " ") + fmt.Printf(" ") } rule := rul3s[node.pegRule] quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) if !pretty { - fmt.Fprintf(w, "%v %v\n", rule, quote) + fmt.Printf("%v %v\n", rule, quote) } else { - fmt.Fprintf(w, "\x1B[34m%v\x1B[m %v\n", rule, quote) + fmt.Printf("\x1B[34m%v\x1B[m %v\n", rule, quote) } if node.up != nil { print(node.up, depth+1) @@ -268,12 +267,12 @@ func (node *node32) print(w io.Writer, pretty bool, buffer string) { print(node, 0) } -func (node *node32) Print(w io.Writer, buffer string) { - node.print(w, false, buffer) +func (node *node32) Print(buffer string) { + node.print(false, buffer) } -func (node *node32) PrettyPrint(w io.Writer, buffer string) { - node.print(w, true, buffer) +func (node *node32) PrettyPrint(buffer string) { + node.print(true, buffer) } type tokens32 struct { @@ -316,24 +315,24 @@ func (t *tokens32) AST() *node32 { } func (t *tokens32) PrintSyntaxTree(buffer string) { - t.AST().Print(os.Stdout, buffer) -} - -func (t *tokens32) WriteSyntaxTree(w io.Writer, buffer string) { - t.AST().Print(w, buffer) + t.AST().Print(buffer) } func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { - t.AST().PrettyPrint(os.Stdout, buffer) + t.AST().PrettyPrint(buffer) } func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { - tree, i := t.tree, int(index) - if i >= len(tree) { - t.tree = append(tree, token32{pegRule: rule, begin: begin, end: end}) - return + if tree := t.tree; int(index) >= len(tree) { + expanded := make([]token32, 2*len(tree)) + copy(expanded, tree) + t.tree = expanded + } + t.tree[index] = token32{ + pegRule: rule, + begin: begin, + end: end, } - tree[i] = token32{pegRule: rule, begin: begin, end: end} } func (t *tokens32) Tokens() []token32 { @@ -397,7 +396,7 @@ type parseError struct { } func (e *parseError) Error() string { - tokens, err := []token32{e.max}, "\n" + tokens, error := []token32{e.max}, "\n" positions, p := make([]int, 2*len(tokens)), 0 for _, token := range tokens { positions[p], p = int(token.begin), p+1 @@ -410,14 +409,14 @@ func (e *parseError) Error() string { } for _, token := range tokens { begin, end := int(token.begin), int(token.end) - err += fmt.Sprintf(format, + error += fmt.Sprintf(format, rul3s[token.pegRule], translations[begin].line, translations[begin].symbol, translations[end].line, translations[end].symbol, strconv.Quote(string(e.p.buffer[begin:end]))) } - return err + return error } func (p *PQL) PrintSyntaxTree() { @@ -428,10 +427,6 @@ func (p *PQL) PrintSyntaxTree() { } } -func (p *PQL) WriteSyntaxTree(w io.Writer) { - p.tokens32.WriteSyntaxTree(w, p.Buffer) -} - func (p *PQL) Execute() { buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0 for _, token := range p.Tokens() { @@ -530,15 +525,15 @@ func (p *PQL) Execute() { case ruleAction43: p.addVal(buffer[begin:end]) case ruleAction44: - p.startCall(buffer[begin:end]) + p.startCall(string(_buffer[begin:end])) case ruleAction45: p.addVal(p.endCall()) case ruleAction46: - p.addVal(buffer[begin:end]) + p.addVal(string(_buffer[begin:end])) case ruleAction47: - p.addVal(buffer[begin:end]) + p.addVal(string(_buffer[begin:end])) case ruleAction48: - p.addVal(buffer[begin:end]) + p.addVal(string(_buffer[begin:end])) case ruleAction49: p.addNumVal(buffer[begin:end], true) case ruleAction50: @@ -571,31 +566,12 @@ func (p *PQL) Execute() { _, _, _, _, _ = buffer, _buffer, text, begin, end } -func Pretty(pretty bool) func(*PQL) error { - return func(p *PQL) error { - p.Pretty = pretty - return nil - } -} - -func Size(size int) func(*PQL) error { - return func(p *PQL) error { - p.tokens32 = tokens32{tree: make([]token32, 0, size)} - return nil - } -} -func (p *PQL) Init(options ...func(*PQL) error) error { +func (p *PQL) Init() { var ( max token32 position, tokenIndex uint32 buffer []rune ) - for _, option := range options { - err := option(p) - if err != nil { - return err - } - } p.reset = func() { max = token32{} position, tokenIndex = 0, 0 @@ -609,7 +585,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { p.reset() _rules := p.rules - tree := p.tokens32 + tree := tokens32{tree: make([]token32, math.MaxInt16)} p.parse = func(rule ...int) error { r := 1 if len(rule) > 0 { @@ -3631,15 +3607,15 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 86 Action43 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 87 Action44 <- <{ p.startCall(buffer[begin:end]) }> */ + /* 87 Action44 <- <{ p.startCall(string(_buffer[begin:end])) }> */ nil, /* 88 Action45 <- <{ p.addVal(p.endCall()) }> */ nil, - /* 89 Action46 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 89 Action46 <- <{ p.addVal(string(_buffer[begin:end])) }> */ nil, - /* 90 Action47 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 90 Action47 <- <{ p.addVal(string(_buffer[begin:end])) }> */ nil, - /* 91 Action48 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 91 Action48 <- <{ p.addVal(string(_buffer[begin:end])) }> */ nil, /* 92 Action49 <- <{ p.addNumVal(buffer[begin:end], true) }> */ nil, @@ -3669,5 +3645,4 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, } p.rules = _rules - return nil } diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 4ebf1dcc9..e6aba444d 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -24,9 +24,7 @@ import ( func TestPEG(t *testing.T) { p := PQL{Buffer: ` SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} - if err := p.Init(); err != nil { - t.Fatalf("initialization error: %v", err) - } + p.Init() err := p.Parse() if err != nil { t.Fatalf("parse error: %v", err) @@ -34,9 +32,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9 p.Execute() p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`} - if err := p.Init(); err != nil { - t.Fatalf("initialization error: %v", err) - } + p.Init() err = p.Parse() if err == nil { t.Fatalf("should have been an error because of the interior unescaped double quote")