mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 17:15:56 +00:00
Use rune slice in all cases, add tests
This commit is contained in:
Alan Bernstein
parent
0a96043c23
commit
6fc5465aaa
4 changed files with 125 additions and 92 deletions
|
|
@ -193,26 +193,6 @@ func TestParser_Parse(t *testing.T) {
|
|||
t.Fatalf("unexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse unicode keys
|
||||
t.Run("UnicodeKey", func(t *testing.T) {
|
||||
// s := `<60><><EFBFBD>t`
|
||||
s := `Æ`
|
||||
q, err := pql.ParseString(`Row(unicode="` + s + `")`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "Row",
|
||||
Args: map[string]interface{}{
|
||||
"unicode": s,
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("uexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func TestUnquote(t *testing.T) {
|
||||
|
|
|
|||
46
pql/pql.peg
46
pql/pql.peg
|
|
@ -14,8 +14,8 @@ Call <- 'Set' {p.startCall("Set")} open col comma dargs (comma timestamp)? clos
|
|||
/ 'Store' {p.startCall("Store")} open Call comma darg close {p.endCall()}
|
||||
/ 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
|
||||
/ 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()}
|
||||
/ 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()}
|
||||
/ < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }
|
||||
/ 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(text)} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(text)} close {p.endCall()}
|
||||
/ < IDENT > { p.startCall(text ) } open allargs comma? close { p.endCall() }
|
||||
allargs <- Call (comma Call)* (comma dargs)? / dargs / sp
|
||||
fargs <- farg (comma fargs)? sp
|
||||
farg <- ( field sp '=' sp fvalue
|
||||
|
|
@ -37,9 +37,9 @@ COND <- ( '><' { p.addBTWN() }
|
|||
)
|
||||
|
||||
conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()}
|
||||
condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(buffer[begin:end])}
|
||||
condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])}
|
||||
condfield <- <fieldExpr> sp {p.condAdd(buffer[begin:end])}
|
||||
condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(text)}
|
||||
condLT <- <('<=' / '<')> sp {p.condAdd(text)}
|
||||
condfield <- <fieldExpr> sp {p.condAdd(text)}
|
||||
|
||||
dvalue <- ( ditem
|
||||
/ lbrack { p.startList() } dlist rbrack { p.endList() }
|
||||
|
|
@ -60,35 +60,35 @@ fitem <- ( itema
|
|||
itema <- ( 'null' &(comma / sp close) { p.addVal(nil) }
|
||||
/ 'true' &(comma / sp close) { p.addVal(true) }
|
||||
/ 'false' &(comma / sp close) { p.addVal(false) }
|
||||
/ timestampfmt { p.addVal(buffer[begin:end]) }
|
||||
/ timestampfmt { p.addVal(text) }
|
||||
)
|
||||
itemb <- ( < IDENT > { p.startCall(string(_buffer[begin:end])) } open allargs comma? close { p.addVal(p.endCall()) }
|
||||
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(string(_buffer[begin:end])) }
|
||||
/ < '"' doublequotedstring '"' > { p.addVal(string(_buffer[begin:end])) }
|
||||
/ < '\'' singlequotedstring '\'' > { p.addVal(string(_buffer[begin:end])) }
|
||||
itemb <- ( < IDENT > { p.startCall(text) } open allargs comma? close { p.addVal(p.endCall()) }
|
||||
/ < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(text) }
|
||||
/ < '"' doublequotedstring '"' > { p.addVal(text) }
|
||||
/ < '\'' singlequotedstring '\'' > { p.addVal(text) }
|
||||
)
|
||||
float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) }
|
||||
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) }
|
||||
float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(text, true) }
|
||||
/ < '-'? '.'[0-9]+ > { p.addNumVal(text, true) }
|
||||
)
|
||||
decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false) }
|
||||
/ < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) }
|
||||
decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(text, false) }
|
||||
/ < '-'? '.'[0-9]+ > { p.addNumVal(text, false) }
|
||||
)
|
||||
|
||||
doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )*
|
||||
singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )*
|
||||
|
||||
fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )*
|
||||
field <- <fieldExpr / reserved> { p.addField(buffer[begin:end]) }
|
||||
field <- <fieldExpr / reserved> { p.addField(text) }
|
||||
reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field')
|
||||
posfield <- <fieldExpr> { p.addPosStr("_field", buffer[begin:end]) }
|
||||
posfield <- <fieldExpr> { p.addPosStr("_field", text) }
|
||||
uint <- [1-9] [0-9]* / '0'
|
||||
col <- ( <uint> {p.addPosNum("_col", buffer[begin:end])}
|
||||
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])}
|
||||
/ < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])}
|
||||
col <- ( <uint> {p.addPosNum("_col", text)}
|
||||
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)}
|
||||
/ < '"' doublequotedstring '"' > {p.addPosStr("_col", text)}
|
||||
)
|
||||
row <- ( <uint> {p.addPosNum("_row", buffer[begin:end])}
|
||||
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])}
|
||||
/ < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])}
|
||||
row <- ( <uint> {p.addPosNum("_row", text)}
|
||||
/ < '\'' singlequotedstring '\'' > {p.addPosStr("_row", text)}
|
||||
/ < '"' doublequotedstring '"' > {p.addPosStr("_row", text)}
|
||||
)
|
||||
|
||||
open <- '(' sp
|
||||
|
|
@ -102,4 +102,4 @@ IDENT <- [[A-Z]] ([[A-Z]] / [0-9])*
|
|||
|
||||
timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9]
|
||||
timestampfmt <- '"' <timestampbasicfmt> '"' / '\'' <timestampbasicfmt> '\'' / <timestampbasicfmt>
|
||||
timestamp <- <timestampfmt> {p.addPosStr("_timestamp", buffer[begin:end])}
|
||||
timestamp <- <timestampfmt> {p.addPosStr("_timestamp", text)}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
package pql
|
||||
|
||||
//go:generate peg -inline pql.peg
|
||||
// Code generated by peg -inline pql.peg DO NOT EDIT.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
|
@ -473,15 +473,15 @@ func (p *PQL) Execute() {
|
|||
case ruleAction17:
|
||||
p.addField("from")
|
||||
case ruleAction18:
|
||||
p.addVal(buffer[begin:end])
|
||||
p.addVal(text)
|
||||
case ruleAction19:
|
||||
p.addField("to")
|
||||
case ruleAction20:
|
||||
p.addVal(buffer[begin:end])
|
||||
p.addVal(text)
|
||||
case ruleAction21:
|
||||
p.endCall()
|
||||
case ruleAction22:
|
||||
p.startCall(buffer[begin:end])
|
||||
p.startCall(text)
|
||||
case ruleAction23:
|
||||
p.endCall()
|
||||
case ruleAction24:
|
||||
|
|
@ -503,11 +503,11 @@ func (p *PQL) Execute() {
|
|||
case ruleAction32:
|
||||
p.endConditional()
|
||||
case ruleAction33:
|
||||
p.condAdd(buffer[begin:end])
|
||||
p.condAdd(text)
|
||||
case ruleAction34:
|
||||
p.condAdd(buffer[begin:end])
|
||||
p.condAdd(text)
|
||||
case ruleAction35:
|
||||
p.condAdd(buffer[begin:end])
|
||||
p.condAdd(text)
|
||||
case ruleAction36:
|
||||
p.startList()
|
||||
case ruleAction37:
|
||||
|
|
@ -523,43 +523,43 @@ func (p *PQL) Execute() {
|
|||
case ruleAction42:
|
||||
p.addVal(false)
|
||||
case ruleAction43:
|
||||
p.addVal(buffer[begin:end])
|
||||
p.addVal(text)
|
||||
case ruleAction44:
|
||||
p.startCall(string(_buffer[begin:end]))
|
||||
p.startCall(text)
|
||||
case ruleAction45:
|
||||
p.addVal(p.endCall())
|
||||
case ruleAction46:
|
||||
p.addVal(string(_buffer[begin:end]))
|
||||
p.addVal(text)
|
||||
case ruleAction47:
|
||||
p.addVal(string(_buffer[begin:end]))
|
||||
p.addVal(text)
|
||||
case ruleAction48:
|
||||
p.addVal(string(_buffer[begin:end]))
|
||||
p.addVal(text)
|
||||
case ruleAction49:
|
||||
p.addNumVal(buffer[begin:end], true)
|
||||
p.addNumVal(text, true)
|
||||
case ruleAction50:
|
||||
p.addNumVal(buffer[begin:end], true)
|
||||
p.addNumVal(text, true)
|
||||
case ruleAction51:
|
||||
p.addNumVal(buffer[begin:end], false)
|
||||
p.addNumVal(text, false)
|
||||
case ruleAction52:
|
||||
p.addNumVal(buffer[begin:end], false)
|
||||
p.addNumVal(text, false)
|
||||
case ruleAction53:
|
||||
p.addField(buffer[begin:end])
|
||||
p.addField(text)
|
||||
case ruleAction54:
|
||||
p.addPosStr("_field", buffer[begin:end])
|
||||
p.addPosStr("_field", text)
|
||||
case ruleAction55:
|
||||
p.addPosNum("_col", buffer[begin:end])
|
||||
p.addPosNum("_col", text)
|
||||
case ruleAction56:
|
||||
p.addPosStr("_col", buffer[begin:end])
|
||||
p.addPosStr("_col", text)
|
||||
case ruleAction57:
|
||||
p.addPosStr("_col", buffer[begin:end])
|
||||
p.addPosStr("_col", text)
|
||||
case ruleAction58:
|
||||
p.addPosNum("_row", buffer[begin:end])
|
||||
p.addPosNum("_row", text)
|
||||
case ruleAction59:
|
||||
p.addPosStr("_row", buffer[begin:end])
|
||||
p.addPosStr("_row", text)
|
||||
case ruleAction60:
|
||||
p.addPosStr("_row", buffer[begin:end])
|
||||
p.addPosStr("_row", text)
|
||||
case ruleAction61:
|
||||
p.addPosStr("_timestamp", buffer[begin:end])
|
||||
p.addPosStr("_timestamp", text)
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -3554,16 +3554,16 @@ func (p *PQL) Init() {
|
|||
nil,
|
||||
/* 59 Action17 <- <{p.addField("from")}> */
|
||||
nil,
|
||||
/* 60 Action18 <- <{p.addVal(buffer[begin:end])}> */
|
||||
/* 60 Action18 <- <{p.addVal(text)}> */
|
||||
nil,
|
||||
/* 61 Action19 <- <{p.addField("to")}> */
|
||||
nil,
|
||||
/* 62 Action20 <- <{p.addVal(buffer[begin:end])}> */
|
||||
/* 62 Action20 <- <{p.addVal(text)}> */
|
||||
nil,
|
||||
/* 63 Action21 <- <{p.endCall()}> */
|
||||
nil,
|
||||
nil,
|
||||
/* 65 Action22 <- <{ p.startCall(buffer[begin:end] ) }> */
|
||||
/* 65 Action22 <- <{ p.startCall(text ) }> */
|
||||
nil,
|
||||
/* 66 Action23 <- <{ p.endCall() }> */
|
||||
nil,
|
||||
|
|
@ -3585,11 +3585,11 @@ func (p *PQL) Init() {
|
|||
nil,
|
||||
/* 75 Action32 <- <{p.endConditional()}> */
|
||||
nil,
|
||||
/* 76 Action33 <- <{p.condAdd(buffer[begin:end])}> */
|
||||
/* 76 Action33 <- <{p.condAdd(text)}> */
|
||||
nil,
|
||||
/* 77 Action34 <- <{p.condAdd(buffer[begin:end])}> */
|
||||
/* 77 Action34 <- <{p.condAdd(text)}> */
|
||||
nil,
|
||||
/* 78 Action35 <- <{p.condAdd(buffer[begin:end])}> */
|
||||
/* 78 Action35 <- <{p.condAdd(text)}> */
|
||||
nil,
|
||||
/* 79 Action36 <- <{ p.startList() }> */
|
||||
nil,
|
||||
|
|
@ -3605,43 +3605,43 @@ func (p *PQL) Init() {
|
|||
nil,
|
||||
/* 85 Action42 <- <{ p.addVal(false) }> */
|
||||
nil,
|
||||
/* 86 Action43 <- <{ p.addVal(buffer[begin:end]) }> */
|
||||
/* 86 Action43 <- <{ p.addVal(text) }> */
|
||||
nil,
|
||||
/* 87 Action44 <- <{ p.startCall(string(_buffer[begin:end])) }> */
|
||||
/* 87 Action44 <- <{ p.startCall(text) }> */
|
||||
nil,
|
||||
/* 88 Action45 <- <{ p.addVal(p.endCall()) }> */
|
||||
nil,
|
||||
/* 89 Action46 <- <{ p.addVal(string(_buffer[begin:end])) }> */
|
||||
/* 89 Action46 <- <{ p.addVal(text) }> */
|
||||
nil,
|
||||
/* 90 Action47 <- <{ p.addVal(string(_buffer[begin:end])) }> */
|
||||
/* 90 Action47 <- <{ p.addVal(text) }> */
|
||||
nil,
|
||||
/* 91 Action48 <- <{ p.addVal(string(_buffer[begin:end])) }> */
|
||||
/* 91 Action48 <- <{ p.addVal(text) }> */
|
||||
nil,
|
||||
/* 92 Action49 <- <{ p.addNumVal(buffer[begin:end], true) }> */
|
||||
/* 92 Action49 <- <{ p.addNumVal(text, true) }> */
|
||||
nil,
|
||||
/* 93 Action50 <- <{ p.addNumVal(buffer[begin:end], true) }> */
|
||||
/* 93 Action50 <- <{ p.addNumVal(text, true) }> */
|
||||
nil,
|
||||
/* 94 Action51 <- <{ p.addNumVal(buffer[begin:end], false) }> */
|
||||
/* 94 Action51 <- <{ p.addNumVal(text, false) }> */
|
||||
nil,
|
||||
/* 95 Action52 <- <{ p.addNumVal(buffer[begin:end], false) }> */
|
||||
/* 95 Action52 <- <{ p.addNumVal(text, false) }> */
|
||||
nil,
|
||||
/* 96 Action53 <- <{ p.addField(buffer[begin:end]) }> */
|
||||
/* 96 Action53 <- <{ p.addField(text) }> */
|
||||
nil,
|
||||
/* 97 Action54 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */
|
||||
/* 97 Action54 <- <{ p.addPosStr("_field", text) }> */
|
||||
nil,
|
||||
/* 98 Action55 <- <{p.addPosNum("_col", buffer[begin:end])}> */
|
||||
/* 98 Action55 <- <{p.addPosNum("_col", text)}> */
|
||||
nil,
|
||||
/* 99 Action56 <- <{p.addPosStr("_col", buffer[begin:end])}> */
|
||||
/* 99 Action56 <- <{p.addPosStr("_col", text)}> */
|
||||
nil,
|
||||
/* 100 Action57 <- <{p.addPosStr("_col", buffer[begin:end])}> */
|
||||
/* 100 Action57 <- <{p.addPosStr("_col", text)}> */
|
||||
nil,
|
||||
/* 101 Action58 <- <{p.addPosNum("_row", buffer[begin:end])}> */
|
||||
/* 101 Action58 <- <{p.addPosNum("_row", text)}> */
|
||||
nil,
|
||||
/* 102 Action59 <- <{p.addPosStr("_row", buffer[begin:end])}> */
|
||||
/* 102 Action59 <- <{p.addPosStr("_row", text)}> */
|
||||
nil,
|
||||
/* 103 Action60 <- <{p.addPosStr("_row", buffer[begin:end])}> */
|
||||
/* 103 Action60 <- <{p.addPosStr("_row", text)}> */
|
||||
nil,
|
||||
/* 104 Action61 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */
|
||||
/* 104 Action61 <- <{p.addPosStr("_timestamp", text)}> */
|
||||
nil,
|
||||
}
|
||||
p.rules = _rules
|
||||
|
|
|
|||
|
|
@ -375,6 +375,48 @@ func TestPQLDeepEquality(t *testing.T) {
|
|||
"_timestamp": "2010-07-08T14:44",
|
||||
},
|
||||
}},
|
||||
{
|
||||
name: "SetWithUnicode",
|
||||
call: `Set(0, unicode="Æ<>漢д ☮♬ ♞🜻💣")`,
|
||||
exp: &Call{
|
||||
Name: "Set",
|
||||
Args: map[string]interface{}{
|
||||
"_col": int64(0),
|
||||
"unicode": `Æ<EFBFBD>漢д ☮♬ ♞🜻💣`,
|
||||
},
|
||||
}},
|
||||
{
|
||||
name: "RowWithUnicode",
|
||||
call: `Row(unicode="Æ<>漢д ☮♬ ♞🜻💣")`,
|
||||
exp: &Call{
|
||||
Name: "Row",
|
||||
Args: map[string]interface{}{
|
||||
"unicode": `Æ<EFBFBD>漢д ☮♬ ♞🜻💣`,
|
||||
},
|
||||
}},
|
||||
{
|
||||
name: "RowsWithUnicode",
|
||||
call: `Rows(job, previous="💣")`,
|
||||
exp: &Call{
|
||||
Name: "Rows",
|
||||
Args: map[string]interface{}{
|
||||
"_field": "job",
|
||||
"previous": `💣`,
|
||||
},
|
||||
}},
|
||||
{
|
||||
name: "TopNWithUnicode",
|
||||
call: `TopN(stargazer, Row(unicode="Æ<>漢д ☮♬ ♞🜻💣"), a="∑")`,
|
||||
exp: &Call{
|
||||
Name: "TopN",
|
||||
Args: map[string]interface{}{
|
||||
"_field": "stargazer",
|
||||
"a": "∑",
|
||||
},
|
||||
Children: []*Call{
|
||||
{Name: "Row", Args: map[string]interface{}{"unicode": "Æ<>漢д ☮♬ ♞🜻💣"}},
|
||||
},
|
||||
}},
|
||||
{
|
||||
name: "SetRowAttrs",
|
||||
call: "SetRowAttrs(myfield, 9, z=4)",
|
||||
|
|
@ -409,6 +451,17 @@ func TestPQLDeepEquality(t *testing.T) {
|
|||
},
|
||||
}},
|
||||
{
|
||||
name: "SetRowAttrsWithUnicodeValues",
|
||||
call: `SetRowAttrs(myfield, "∫", z="∀", a="∑")`,
|
||||
exp: &Call{
|
||||
Name: "SetRowAttrs",
|
||||
Args: map[string]interface{}{
|
||||
"z": "∀",
|
||||
"a": "∑",
|
||||
"_field": "myfield",
|
||||
"_row": "∫",
|
||||
},
|
||||
}}, {
|
||||
name: "SetColumnAttrs",
|
||||
call: "SetColumnAttrs(9, z=4)",
|
||||
exp: &Call{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue