scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 11:27:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use _buffer instead of buffer

Browse source
This commit is contained in:
Alan Bernstein 2020-08-17 22:16:47 -05:00
parent 0a37f396f8
commit 0a96043c23
5 changed files with 60 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pql/parser.go
View file

@ -61,9 +61,7 @@ func (p *parser) Parse() (*Query, error) {
p.PQL = PQL{
Buffer: string(buf),
}
if err := p.Init(); err != nil {
return nil, errors.Wrap(err, "initializing")
}
p.Init()
err = p.PQL.Parse()
if err != nil {
return nil, errors.Wrap(err, "parsing")

19
pql/parser_test.go
View file

@ -194,6 +194,25 @@ func TestParser_Parse(t *testing.T) {
}
})
// Parse unicode keys
t.Run("UnicodeKey", func(t *testing.T) {
// s := `<60><><EFBFBD>t`
s := `Æ`
q, err := pql.ParseString(`Row(unicode="` + s + `")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "Row",
Args: map[string]interface{}{
"unicode": s,
},
},
) {
t.Fatalf("uexpected call: %#v", q.Calls[0])
}
})
}
func TestUnquote(t *testing.T) {

8
pql/pql.peg
View file

@ -62,10 +62,10 @@ itema <- ( 'null' &(comma / sp close) { p.addVal(nil) }
/ 'false' &(comma / sp close) { p.addVal(false) }
/ timestampfmt { p.addVal(buffer[begin:end]) }
)
itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) }
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) }
/ < '"' doublequotedstring '"' > { p.addVal(buffer[begin:end]) }
/ < '\'' singlequotedstring '\'' > { p.addVal(buffer[begin:end]) }
itemb <- ( < IDENT > { p.startCall(string(_buffer[begin:end])) } open allargs comma? close { p.addVal(p.endCall()) }
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(string(_buffer[begin:end])) }
/ < '"' doublequotedstring '"' > { p.addVal(string(_buffer[begin:end])) }
/ < '\'' singlequotedstring '\'' > { p.addVal(string(_buffer[begin:end])) }
)
float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) }
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) }

93
pql/pql.peg.go
View file

@ -1,11 +1,10 @@
package pql
// Code generated by peg -inline pql.peg DO NOT EDIT.
//go:generate peg -inline pql.peg
import (
"fmt"
"io"
"os"
"math"
"sort"
"strconv"
)
@ -245,19 +244,19 @@ type node32 struct {
up, next *node32
}
func (node *node32) print(w io.Writer, pretty bool, buffer string) {
func (node *node32) print(pretty bool, buffer string) {
var print func(node *node32, depth int)
print = func(node *node32, depth int) {
for node != nil {
for c := 0; c < depth; c++ {
fmt.Fprintf(w, " ")
fmt.Printf(" ")
}
rule := rul3s[node.pegRule]
quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end])))
if !pretty {
fmt.Fprintf(w, "%v %v\n", rule, quote)
fmt.Printf("%v %v\n", rule, quote)
} else {
fmt.Fprintf(w, "\x1B[34m%v\x1B[m %v\n", rule, quote)
fmt.Printf("\x1B[34m%v\x1B[m %v\n", rule, quote)
}
if node.up != nil {
print(node.up, depth+1)
@ -268,12 +267,12 @@ func (node *node32) print(w io.Writer, pretty bool, buffer string) {
print(node, 0)
}
func (node *node32) Print(w io.Writer, buffer string) {
node.print(w, false, buffer)
func (node *node32) Print(buffer string) {
node.print(false, buffer)
}
func (node *node32) PrettyPrint(w io.Writer, buffer string) {
node.print(w, true, buffer)
func (node *node32) PrettyPrint(buffer string) {
node.print(true, buffer)
}
type tokens32 struct {
@ -316,24 +315,24 @@ func (t *tokens32) AST() *node32 {
}
func (t *tokens32) PrintSyntaxTree(buffer string) {
t.AST().Print(os.Stdout, buffer)
}
func (t *tokens32) WriteSyntaxTree(w io.Writer, buffer string) {
t.AST().Print(w, buffer)
t.AST().Print(buffer)
}
func (t *tokens32) PrettyPrintSyntaxTree(buffer string) {
t.AST().PrettyPrint(os.Stdout, buffer)
t.AST().PrettyPrint(buffer)
}
func (t *tokens32) Add(rule pegRule, begin, end, index uint32) {
tree, i := t.tree, int(index)
if i >= len(tree) {
t.tree = append(tree, token32{pegRule: rule, begin: begin, end: end})
return
if tree := t.tree; int(index) >= len(tree) {
expanded := make([]token32, 2*len(tree))
copy(expanded, tree)
t.tree = expanded
}
t.tree[index] = token32{
pegRule: rule,
begin: begin,
end: end,
}
tree[i] = token32{pegRule: rule, begin: begin, end: end}
}
func (t *tokens32) Tokens() []token32 {
@ -397,7 +396,7 @@ type parseError struct {
}
func (e *parseError) Error() string {
tokens, err := []token32{e.max}, "\n"
tokens, error := []token32{e.max}, "\n"
positions, p := make([]int, 2*len(tokens)), 0
for _, token := range tokens {
positions[p], p = int(token.begin), p+1
@ -410,14 +409,14 @@ func (e *parseError) Error() string {
}
for _, token := range tokens {
begin, end := int(token.begin), int(token.end)
err += fmt.Sprintf(format,
error += fmt.Sprintf(format,
rul3s[token.pegRule],
translations[begin].line, translations[begin].symbol,
translations[end].line, translations[end].symbol,
strconv.Quote(string(e.p.buffer[begin:end])))
}
return err
return error
}
func (p *PQL) PrintSyntaxTree() {
@ -428,10 +427,6 @@ func (p *PQL) PrintSyntaxTree() {
}
}
func (p *PQL) WriteSyntaxTree(w io.Writer) {
p.tokens32.WriteSyntaxTree(w, p.Buffer)
}
func (p *PQL) Execute() {
buffer, _buffer, text, begin, end := p.Buffer, p.buffer, "", 0, 0
for _, token := range p.Tokens() {
@ -530,15 +525,15 @@ func (p *PQL) Execute() {
case ruleAction43:
p.addVal(buffer[begin:end])
case ruleAction44:
p.startCall(buffer[begin:end])
p.startCall(string(_buffer[begin:end]))
case ruleAction45:
p.addVal(p.endCall())
case ruleAction46:
p.addVal(buffer[begin:end])
p.addVal(string(_buffer[begin:end]))
case ruleAction47:
p.addVal(buffer[begin:end])
p.addVal(string(_buffer[begin:end]))
case ruleAction48:
p.addVal(buffer[begin:end])
p.addVal(string(_buffer[begin:end]))
case ruleAction49:
p.addNumVal(buffer[begin:end], true)
case ruleAction50:
@ -571,31 +566,12 @@ func (p *PQL) Execute() {
_, _, _, _, _ = buffer, _buffer, text, begin, end
}
func Pretty(pretty bool) func(*PQL) error {
return func(p *PQL) error {
p.Pretty = pretty
return nil
}
}
func Size(size int) func(*PQL) error {
return func(p *PQL) error {
p.tokens32 = tokens32{tree: make([]token32, 0, size)}
return nil
}
}
func (p *PQL) Init(options ...func(*PQL) error) error {
func (p *PQL) Init() {
var (
max token32
position, tokenIndex uint32
buffer []rune
)
for _, option := range options {
err := option(p)
if err != nil {
return err
}
}
p.reset = func() {
max = token32{}
position, tokenIndex = 0, 0
@ -609,7 +585,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error {
p.reset()
_rules := p.rules
tree := p.tokens32
tree := tokens32{tree: make([]token32, math.MaxInt16)}
p.parse = func(rule ...int) error {
r := 1
if len(rule) > 0 {
@ -3631,15 +3607,15 @@ func (p *PQL) Init(options ...func(*PQL) error) error {
nil,
/* 86 Action43 <- <{ p.addVal(buffer[begin:end]) }> */
nil,
/* 87 Action44 <- <{ p.startCall(buffer[begin:end]) }> */
/* 87 Action44 <- <{ p.startCall(string(_buffer[begin:end])) }> */
nil,
/* 88 Action45 <- <{ p.addVal(p.endCall()) }> */
nil,
/* 89 Action46 <- <{ p.addVal(buffer[begin:end]) }> */
/* 89 Action46 <- <{ p.addVal(string(_buffer[begin:end])) }> */
nil,
/* 90 Action47 <- <{ p.addVal(buffer[begin:end]) }> */
/* 90 Action47 <- <{ p.addVal(string(_buffer[begin:end])) }> */
nil,
/* 91 Action48 <- <{ p.addVal(buffer[begin:end]) }> */
/* 91 Action48 <- <{ p.addVal(string(_buffer[begin:end])) }> */
nil,
/* 92 Action49 <- <{ p.addNumVal(buffer[begin:end], true) }> */
nil,
@ -3669,5 +3645,4 @@ func (p *PQL) Init(options ...func(*PQL) error) error {
nil,
}
p.rules = _rules
return nil
}

8
pql/pqlpeg_test.go
View file

@ -24,9 +24,7 @@ import (
func TestPEG(t *testing.T) {
p := PQL{Buffer: `
SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]}
if err := p.Init(); err != nil {
t.Fatalf("initialization error: %v", err)
}
p.Init()
err := p.Parse()
if err != nil {
t.Fatalf("parse error: %v", err)
@ -34,9 +32,7 @@ SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9
p.Execute()
p = PQL{Buffer: `SetRowAttrs(attr="http://zoo9.com=\\'hello' "and \"hello\"")`}
if err := p.Init(); err != nil {
t.Fatalf("initialization error: %v", err)
}
p.Init()
err = p.Parse()
if err == nil {
t.Fatalf("should have been an error because of the interior unescaped double quote")