From 6fc5465aaa0f8fb8f08d5dfa8a764c0689c5c339 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 19 Aug 2020 16:19:43 -0500 Subject: [PATCH] Use rune slice in all cases, add tests --- pql/parser_test.go | 20 ---------- pql/pql.peg | 46 +++++++++++----------- pql/pql.peg.go | 98 +++++++++++++++++++++++----------------------- pql/pqlpeg_test.go | 53 +++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 92 deletions(-) diff --git a/pql/parser_test.go b/pql/parser_test.go index 94d97d1a1..d2ee45748 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -193,26 +193,6 @@ func TestParser_Parse(t *testing.T) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) - - // Parse unicode keys - t.Run("UnicodeKey", func(t *testing.T) { - // s := `���t` - s := `Æ` - q, err := pql.ParseString(`Row(unicode="` + s + `")`) - if err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(q.Calls[0], - &pql.Call{ - Name: "Row", - Args: map[string]interface{}{ - "unicode": s, - }, - }, - ) { - t.Fatalf("uexpected call: %#v", q.Calls[0]) - } - }) - } func TestUnquote(t *testing.T) { diff --git a/pql/pql.peg b/pql/pql.peg index dcb10d26d..b58e36723 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -14,8 +14,8 @@ Call <- 'Set' {p.startCall("Set")} open col comma dargs (comma timestamp)? clos / 'Store' {p.startCall("Store")} open Call comma darg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()} - / 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()} - / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } + / 'Range' {p.startCall("Range")} open field sp '=' sp fvalue comma 'from='? {p.addField("from")} timestampfmt {p.addVal(text)} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(text)} close {p.endCall()} + / < IDENT > { p.startCall(text ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma dargs)? / dargs / sp fargs <- farg (comma fargs)? sp farg <- ( field sp '=' sp fvalue @@ -37,9 +37,9 @@ COND <- ( '><' { p.addBTWN() } ) conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()} -condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(buffer[begin:end])} -condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])} -condfield <- sp {p.condAdd(buffer[begin:end])} +condint <- < '-'? [0-9]* '.' [0-9]+ / '0' / '-'? [1-9] [0-9]* > sp {p.condAdd(text)} +condLT <- <('<=' / '<')> sp {p.condAdd(text)} +condfield <- sp {p.condAdd(text)} dvalue <- ( ditem / lbrack { p.startList() } dlist rbrack { p.endList() } @@ -60,35 +60,35 @@ fitem <- ( itema itema <- ( 'null' &(comma / sp close) { p.addVal(nil) } / 'true' &(comma / sp close) { p.addVal(true) } / 'false' &(comma / sp close) { p.addVal(false) } - / timestampfmt { p.addVal(buffer[begin:end]) } + / timestampfmt { p.addVal(text) } ) -itemb <- ( < IDENT > { p.startCall(string(_buffer[begin:end])) } open allargs comma? close { p.addVal(p.endCall()) } - / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(string(_buffer[begin:end])) } - / < '"' doublequotedstring '"' > { p.addVal(string(_buffer[begin:end])) } - / < '\'' singlequotedstring '\'' > { p.addVal(string(_buffer[begin:end])) } +itemb <- ( < IDENT > { p.startCall(text) } open allargs comma? close { p.addVal(p.endCall()) } + / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(text) } + / < '"' doublequotedstring '"' > { p.addVal(text) } + / < '\'' singlequotedstring '\'' > { p.addVal(text) } ) -float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) } - / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) } +float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(text, true) } + / < '-'? '.'[0-9]+ > { p.addNumVal(text, true) } ) -decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false) } - / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) } +decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(text, false) } + / < '-'? '.'[0-9]+ > { p.addNumVal(text, false) } ) doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )* singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* -field <- { p.addField(buffer[begin:end]) } +field <- { p.addField(text) } reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field') -posfield <- { p.addPosStr("_field", buffer[begin:end]) } +posfield <- { p.addPosStr("_field", text) } uint <- [1-9] [0-9]* / '0' -col <- ( {p.addPosNum("_col", buffer[begin:end])} - / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])} - / < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])} +col <- ( {p.addPosNum("_col", text)} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", text)} + / < '"' doublequotedstring '"' > {p.addPosStr("_col", text)} ) -row <- ( {p.addPosNum("_row", buffer[begin:end])} - / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])} - / < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])} +row <- ( {p.addPosNum("_row", text)} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", text)} + / < '"' doublequotedstring '"' > {p.addPosStr("_row", text)} ) open <- '(' sp @@ -102,4 +102,4 @@ IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] timestampfmt <- '"' '"' / '\'' '\'' / -timestamp <- {p.addPosStr("_timestamp", buffer[begin:end])} +timestamp <- {p.addPosStr("_timestamp", text)} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index e16002005..3c2a70940 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -1,6 +1,6 @@ package pql -//go:generate peg -inline pql.peg +// Code generated by peg -inline pql.peg DO NOT EDIT. import ( "fmt" @@ -473,15 +473,15 @@ func (p *PQL) Execute() { case ruleAction17: p.addField("from") case ruleAction18: - p.addVal(buffer[begin:end]) + p.addVal(text) case ruleAction19: p.addField("to") case ruleAction20: - p.addVal(buffer[begin:end]) + p.addVal(text) case ruleAction21: p.endCall() case ruleAction22: - p.startCall(buffer[begin:end]) + p.startCall(text) case ruleAction23: p.endCall() case ruleAction24: @@ -503,11 +503,11 @@ func (p *PQL) Execute() { case ruleAction32: p.endConditional() case ruleAction33: - p.condAdd(buffer[begin:end]) + p.condAdd(text) case ruleAction34: - p.condAdd(buffer[begin:end]) + p.condAdd(text) case ruleAction35: - p.condAdd(buffer[begin:end]) + p.condAdd(text) case ruleAction36: p.startList() case ruleAction37: @@ -523,43 +523,43 @@ func (p *PQL) Execute() { case ruleAction42: p.addVal(false) case ruleAction43: - p.addVal(buffer[begin:end]) + p.addVal(text) case ruleAction44: - p.startCall(string(_buffer[begin:end])) + p.startCall(text) case ruleAction45: p.addVal(p.endCall()) case ruleAction46: - p.addVal(string(_buffer[begin:end])) + p.addVal(text) case ruleAction47: - p.addVal(string(_buffer[begin:end])) + p.addVal(text) case ruleAction48: - p.addVal(string(_buffer[begin:end])) + p.addVal(text) case ruleAction49: - p.addNumVal(buffer[begin:end], true) + p.addNumVal(text, true) case ruleAction50: - p.addNumVal(buffer[begin:end], true) + p.addNumVal(text, true) case ruleAction51: - p.addNumVal(buffer[begin:end], false) + p.addNumVal(text, false) case ruleAction52: - p.addNumVal(buffer[begin:end], false) + p.addNumVal(text, false) case ruleAction53: - p.addField(buffer[begin:end]) + p.addField(text) case ruleAction54: - p.addPosStr("_field", buffer[begin:end]) + p.addPosStr("_field", text) case ruleAction55: - p.addPosNum("_col", buffer[begin:end]) + p.addPosNum("_col", text) case ruleAction56: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_col", text) case ruleAction57: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_col", text) case ruleAction58: - p.addPosNum("_row", buffer[begin:end]) + p.addPosNum("_row", text) case ruleAction59: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_row", text) case ruleAction60: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_row", text) case ruleAction61: - p.addPosStr("_timestamp", buffer[begin:end]) + p.addPosStr("_timestamp", text) } } @@ -3554,16 +3554,16 @@ func (p *PQL) Init() { nil, /* 59 Action17 <- <{p.addField("from")}> */ nil, - /* 60 Action18 <- <{p.addVal(buffer[begin:end])}> */ + /* 60 Action18 <- <{p.addVal(text)}> */ nil, /* 61 Action19 <- <{p.addField("to")}> */ nil, - /* 62 Action20 <- <{p.addVal(buffer[begin:end])}> */ + /* 62 Action20 <- <{p.addVal(text)}> */ nil, /* 63 Action21 <- <{p.endCall()}> */ nil, nil, - /* 65 Action22 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 65 Action22 <- <{ p.startCall(text ) }> */ nil, /* 66 Action23 <- <{ p.endCall() }> */ nil, @@ -3585,11 +3585,11 @@ func (p *PQL) Init() { nil, /* 75 Action32 <- <{p.endConditional()}> */ nil, - /* 76 Action33 <- <{p.condAdd(buffer[begin:end])}> */ + /* 76 Action33 <- <{p.condAdd(text)}> */ nil, - /* 77 Action34 <- <{p.condAdd(buffer[begin:end])}> */ + /* 77 Action34 <- <{p.condAdd(text)}> */ nil, - /* 78 Action35 <- <{p.condAdd(buffer[begin:end])}> */ + /* 78 Action35 <- <{p.condAdd(text)}> */ nil, /* 79 Action36 <- <{ p.startList() }> */ nil, @@ -3605,43 +3605,43 @@ func (p *PQL) Init() { nil, /* 85 Action42 <- <{ p.addVal(false) }> */ nil, - /* 86 Action43 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 86 Action43 <- <{ p.addVal(text) }> */ nil, - /* 87 Action44 <- <{ p.startCall(string(_buffer[begin:end])) }> */ + /* 87 Action44 <- <{ p.startCall(text) }> */ nil, /* 88 Action45 <- <{ p.addVal(p.endCall()) }> */ nil, - /* 89 Action46 <- <{ p.addVal(string(_buffer[begin:end])) }> */ + /* 89 Action46 <- <{ p.addVal(text) }> */ nil, - /* 90 Action47 <- <{ p.addVal(string(_buffer[begin:end])) }> */ + /* 90 Action47 <- <{ p.addVal(text) }> */ nil, - /* 91 Action48 <- <{ p.addVal(string(_buffer[begin:end])) }> */ + /* 91 Action48 <- <{ p.addVal(text) }> */ nil, - /* 92 Action49 <- <{ p.addNumVal(buffer[begin:end], true) }> */ + /* 92 Action49 <- <{ p.addNumVal(text, true) }> */ nil, - /* 93 Action50 <- <{ p.addNumVal(buffer[begin:end], true) }> */ + /* 93 Action50 <- <{ p.addNumVal(text, true) }> */ nil, - /* 94 Action51 <- <{ p.addNumVal(buffer[begin:end], false) }> */ + /* 94 Action51 <- <{ p.addNumVal(text, false) }> */ nil, - /* 95 Action52 <- <{ p.addNumVal(buffer[begin:end], false) }> */ + /* 95 Action52 <- <{ p.addNumVal(text, false) }> */ nil, - /* 96 Action53 <- <{ p.addField(buffer[begin:end]) }> */ + /* 96 Action53 <- <{ p.addField(text) }> */ nil, - /* 97 Action54 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 97 Action54 <- <{ p.addPosStr("_field", text) }> */ nil, - /* 98 Action55 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 98 Action55 <- <{p.addPosNum("_col", text)}> */ nil, - /* 99 Action56 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 99 Action56 <- <{p.addPosStr("_col", text)}> */ nil, - /* 100 Action57 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 100 Action57 <- <{p.addPosStr("_col", text)}> */ nil, - /* 101 Action58 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 101 Action58 <- <{p.addPosNum("_row", text)}> */ nil, - /* 102 Action59 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 102 Action59 <- <{p.addPosStr("_row", text)}> */ nil, - /* 103 Action60 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 103 Action60 <- <{p.addPosStr("_row", text)}> */ nil, - /* 104 Action61 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 104 Action61 <- <{p.addPosStr("_timestamp", text)}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index e6aba444d..9339cad3a 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -375,6 +375,48 @@ func TestPQLDeepEquality(t *testing.T) { "_timestamp": "2010-07-08T14:44", }, }}, + { + name: "SetWithUnicode", + call: `Set(0, unicode="Æ�漢д ☮♬ ♞🜻💣")`, + exp: &Call{ + Name: "Set", + Args: map[string]interface{}{ + "_col": int64(0), + "unicode": `Æ�漢д ☮♬ ♞🜻💣`, + }, + }}, + { + name: "RowWithUnicode", + call: `Row(unicode="Æ�漢д ☮♬ ♞🜻💣")`, + exp: &Call{ + Name: "Row", + Args: map[string]interface{}{ + "unicode": `Æ�漢д ☮♬ ♞🜻💣`, + }, + }}, + { + name: "RowsWithUnicode", + call: `Rows(job, previous="💣")`, + exp: &Call{ + Name: "Rows", + Args: map[string]interface{}{ + "_field": "job", + "previous": `💣`, + }, + }}, + { + name: "TopNWithUnicode", + call: `TopN(stargazer, Row(unicode="Æ�漢д ☮♬ ♞🜻💣"), a="∑")`, + exp: &Call{ + Name: "TopN", + Args: map[string]interface{}{ + "_field": "stargazer", + "a": "∑", + }, + Children: []*Call{ + {Name: "Row", Args: map[string]interface{}{"unicode": "Æ�漢д ☮♬ ♞🜻💣"}}, + }, + }}, { name: "SetRowAttrs", call: "SetRowAttrs(myfield, 9, z=4)", @@ -409,6 +451,17 @@ func TestPQLDeepEquality(t *testing.T) { }, }}, { + name: "SetRowAttrsWithUnicodeValues", + call: `SetRowAttrs(myfield, "∫", z="∀", a="∑")`, + exp: &Call{ + Name: "SetRowAttrs", + Args: map[string]interface{}{ + "z": "∀", + "a": "∑", + "_field": "myfield", + "_row": "∫", + }, + }}, { name: "SetColumnAttrs", call: "SetColumnAttrs(9, z=4)", exp: &Call{