From 1c1204fc77cfedff7f9173b7460f76cf9e3bcfa2 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 28 May 2020 23:53:28 -0500 Subject: [PATCH] modify PQL parser to handle escapes in string values This modifies the parser to properly "unquote" incoming strings. So if a string comes in double or single quoted, we approximately follow Go rules for removing the quotes and processing escape sequences. The differences from Go are: 1. we only support backslash, quote, tab and newline escape sequenences. 2. Single quoted strings are supported and work just like double quoted strings. 3. The peg parser won't actually accept backquoted strings (I don't think) Fixes: #411 --- pql/ast.go | 7 + pql/parser.go | 78 ++++ pql/parser_test.go | 67 ++++ pql/pql.peg | 16 +- pql/pql.peg.go | 811 +++++++++++++++++++++++------------------- server/server_test.go | 44 +++ 6 files changed, 642 insertions(+), 381 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index e4163a66e..e93a2aaf5 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -134,6 +134,13 @@ func (q *Query) validateArgField(elem *callStackElem) { } func (q *Query) addVal(val interface{}) { + if vs, ok := val.(string); ok { + vsu, err := Unquote(vs) + if err != nil { + panic(err) + } + val = vsu + } elem := q.lastCallStackElem() if elem == nil || elem.lastField == "" { panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) diff --git a/pql/parser.go b/pql/parser.go index b3ad984f9..8a5907dbf 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -18,7 +18,9 @@ import ( "fmt" "io" "io/ioutil" + "strconv" "strings" + "unicode/utf8" "github.com/pkg/errors" ) @@ -95,3 +97,79 @@ func (p *parser) Parse() (*Query, error) { return &p.Query, nil } + +// Unquote interprets s as a single-quoted, double-quoted, or +// backquoted Go string literal, returning the string value that s +// quotes. It is a copy of stdlib's strconv.Unquote, but modified so +// that if s is single-quoted, it can still be a string rather than +// only character literal. This version of Unquote also accepts +// unquoted strings and passes them back unchanged. +func Unquote(s string) (string, error) { + n := len(s) + if n < 2 { + return s, nil + } + quote := s[0] + if quote != '"' && quote != '\'' && quote != '`' { + return s, nil + } + if quote != s[n-1] { + return "", strconv.ErrSyntax + } + s = s[1 : n-1] + + if quote == '`' { + if contains(s, '`') { + return "", strconv.ErrSyntax + } + if contains(s, '\r') { + // -1 because we know there is at least one \r to remove. + buf := make([]byte, 0, len(s)-1) + for i := 0; i < len(s); i++ { + if s[i] != '\r' { + buf = append(buf, s[i]) + } + } + return string(buf), nil + } + return s, nil + } + if quote != '"' && quote != '\'' { + return "", strconv.ErrSyntax + } + if contains(s, '\n') { + return "", strconv.ErrSyntax + } + + // Is it trivial? Avoid allocation. + if !contains(s, '\\') && !contains(s, quote) { + switch quote { + case '"', '\'': + if utf8.ValidString(s) { + return s, nil + } + } + } + + var runeTmp [utf8.UTFMax]byte + buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations. + for len(s) > 0 { + c, multibyte, ss, err := strconv.UnquoteChar(s, quote) + if err != nil { + return "", err + } + s = ss + if c < utf8.RuneSelf || !multibyte { + buf = append(buf, byte(c)) + } else { + n := utf8.EncodeRune(runeTmp[:], c) + buf = append(buf, runeTmp[:n]...) + } + } + return string(buf), nil +} + +// contains reports whether the string contains the byte c. +func contains(s string, c byte) bool { + return strings.ContainsRune(s, rune(c)) +} diff --git a/pql/parser_test.go b/pql/parser_test.go index 54bb9e332..8e5cf8cc4 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -16,6 +16,7 @@ package pql_test import ( "reflect" + "strings" "testing" "github.com/pilosa/pilosa/v2/pql" @@ -192,4 +193,70 @@ func TestParser_Parse(t *testing.T) { t.Fatalf("unexpected call: %#v", q.Calls[0]) } }) + +} + +func TestUnquote(t *testing.T) { + tests := []struct { + name string + value string + exp string + expErr string + }{ + { + name: "simple double", + value: `"hello"`, + exp: "hello", + }, + { + name: "simple single", + value: `'hello'`, + exp: "hello", + }, + { + name: "double with esc", + value: `"he\"llo"`, + exp: "he\"llo", + }, + { + name: "single with esc", + value: `'he\'llo'`, + exp: "he'llo", + }, + { + name: "single with backslash and esc", + value: `'he\\\'llo'`, + exp: `he\'llo`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := pql.Unquote(test.value) + if testErr(t, test.expErr, err) { + return + } + if got != test.exp { + t.Errorf("exp: '%s'\ngot: '%s'", test.exp, got) + } + }) + } + +} + +func testErr(t *testing.T, exp string, actual error) (done bool) { + t.Helper() + if exp == "" && actual == nil { + return false + } + if exp == "" && actual != nil { + t.Fatalf("unexpected error: %v", actual) + } + if exp != "" && actual == nil { + t.Fatalf("expected error like '%s'", exp) + } + if !strings.Contains(actual.Error(), exp) { + t.Fatalf("unmatched errs exp/got\n%s\n%v", exp, actual) + } + return true } diff --git a/pql/pql.peg b/pql/pql.peg index ecb435549..7e6c039bd 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -64,8 +64,8 @@ itema <- ( 'null' &(comma / sp close) { p.addVal(nil) } ) itemb <- ( < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } - / < '"' doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) } - / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) } + / < '"' doublequotedstring '"' > { p.addVal(buffer[begin:end]) } + / < '\'' singlequotedstring '\'' > { p.addVal(buffer[begin:end]) } ) float <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], true) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], true) } @@ -74,8 +74,8 @@ decimal <- ( < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end], false / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end], false) } ) -doublequotedstring <- ( '\\"' / '\\\\' / [^"] )* -singlequotedstring <- ( '\\\'' / '\\\\' / [^'] )* +doublequotedstring <- ( '\\"' / '\\\\' / '\\n' / '\\t' / [^"\\] )* +singlequotedstring <- ( '\\\'' / '\\\\' / '\\n' / '\\t' / [^'\\] )* fieldExpr <- ( [[A-Z]] / '_' ) ( [[A-Z]] / [0-9] / '_' / '-' )* field <- { p.addField(buffer[begin:end]) } @@ -83,12 +83,12 @@ reserved <- ('_row' / '_col' / '_start' / '_end' / '_timestamp' / '_field') posfield <- { p.addPosStr("_field", buffer[begin:end]) } uint <- [1-9] [0-9]* / '0' col <- ( {p.addPosNum("_col", buffer[begin:end])} - / '\'' '\'' {p.addPosStr("_col", buffer[begin:end])} - / '"' '"' {p.addPosStr("_col", buffer[begin:end])} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_col", buffer[begin:end])} + / < '"' doublequotedstring '"' > {p.addPosStr("_col", buffer[begin:end])} ) row <- ( {p.addPosNum("_row", buffer[begin:end])} - / '\'' '\'' {p.addPosStr("_row", buffer[begin:end])} - / '"' '"' {p.addPosStr("_row", buffer[begin:end])} + / < '\'' singlequotedstring '\'' > {p.addPosStr("_row", buffer[begin:end])} + / < '"' doublequotedstring '"' > {p.addPosStr("_row", buffer[begin:end])} ) open <- '(' sp diff --git a/pql/pql.peg.go b/pql/pql.peg.go index c1e2db0c7..3cba1b702 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -536,8 +536,7 @@ func (p *PQL) Execute() { case ruleAction46: p.addVal(buffer[begin:end]) case ruleAction47: - s, _ := strconv.Unquote(buffer[begin:end]) - p.addVal(s) + p.addVal(buffer[begin:end]) case ruleAction48: p.addVal(buffer[begin:end]) case ruleAction49: @@ -836,42 +835,42 @@ func (p *PQL) Init(options ...func(*PQL) error) error { goto l19 l20: position, tokenIndex = position19, tokenIndex19 - if buffer[position] != rune('\'') { - goto l23 - } - position++ { position24 := position + if buffer[position] != rune('\'') { + goto l23 + } + position++ if !_rules[rulesinglequotedstring]() { goto l23 } + if buffer[position] != rune('\'') { + goto l23 + } + position++ add(rulePegText, position24) } - if buffer[position] != rune('\'') { - goto l23 - } - position++ { add(ruleAction59, position) } goto l19 l23: position, tokenIndex = position19, tokenIndex19 - if buffer[position] != rune('"') { - goto l16 - } - position++ { position26 := position + if buffer[position] != rune('"') { + goto l16 + } + position++ if !_rules[ruledoublequotedstring]() { goto l16 } + if buffer[position] != rune('"') { + goto l16 + } + position++ add(rulePegText, position26) } - if buffer[position] != rune('"') { - goto l16 - } - position++ { add(ruleAction60, position) } @@ -2433,7 +2432,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position, tokenIndex = position209, tokenIndex209 return false }, - /* 19 itemb <- <(( Action44 open allargs comma? close Action45) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action46) / (<('"' doublequotedstring '"')> Action47) / ('\'' '\'' Action48))> */ + /* 19 itemb <- <(( Action44 open allargs comma? close Action45) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action46) / (<('"' doublequotedstring '"')> Action47) / (<('\'' singlequotedstring '\'')> Action48))> */ func() bool { position228, tokenIndex228 := position, tokenIndex { @@ -2599,21 +2598,21 @@ func (p *PQL) Init(options ...func(*PQL) error) error { goto l230 l254: position, tokenIndex = position230, tokenIndex230 - if buffer[position] != rune('\'') { - goto l228 - } - position++ { position257 := position + if buffer[position] != rune('\'') { + goto l228 + } + position++ if !_rules[rulesinglequotedstring]() { goto l228 } + if buffer[position] != rune('\'') { + goto l228 + } + position++ add(rulePegText, position257) } - if buffer[position] != rune('\'') { - goto l228 - } - position++ { add(ruleAction48, position) } @@ -2630,7 +2629,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 21 decimal <- <((<('-'? [0-9]+ ('.' [0-9]*)?)> Action51) / (<('-'? '.' [0-9]+)> Action52))> */ nil, - /* 22 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ + /* 22 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('"' / '\\') .))*> */ func() bool { { position262 := position @@ -2660,16 +2659,49 @@ func (p *PQL) Init(options ...func(*PQL) error) error { position++ goto l265 l267: + position, tokenIndex = position265, tokenIndex265 + if buffer[position] != rune('\\') { + goto l268 + } + position++ + if buffer[position] != rune('n') { + goto l268 + } + position++ + goto l265 + l268: + position, tokenIndex = position265, tokenIndex265 + if buffer[position] != rune('\\') { + goto l269 + } + position++ + if buffer[position] != rune('t') { + goto l269 + } + position++ + goto l265 + l269: position, tokenIndex = position265, tokenIndex265 { - position268, tokenIndex268 := position, tokenIndex - if buffer[position] != rune('"') { - goto l268 + position270, tokenIndex270 := position, tokenIndex + { + position271, tokenIndex271 := position, tokenIndex + if buffer[position] != rune('"') { + goto l272 + } + position++ + goto l271 + l272: + position, tokenIndex = position271, tokenIndex271 + if buffer[position] != rune('\\') { + goto l270 + } + position++ } - position++ + l271: goto l264 - l268: - position, tokenIndex = position268, tokenIndex268 + l270: + position, tokenIndex = position270, tokenIndex270 } if !matchDot() { goto l264 @@ -2684,795 +2716,828 @@ func (p *PQL) Init(options ...func(*PQL) error) error { } return true }, - /* 23 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ + /* 23 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / ('\\' 'n') / ('\\' 't') / (!('\'' / '\\') .))*> */ func() bool { { - position270 := position - l271: + position274 := position + l275: { - position272, tokenIndex272 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex { - position273, tokenIndex273 := position, tokenIndex + position277, tokenIndex277 := position, tokenIndex if buffer[position] != rune('\\') { - goto l274 + goto l278 } position++ if buffer[position] != rune('\'') { - goto l274 + goto l278 } position++ - goto l273 - l274: - position, tokenIndex = position273, tokenIndex273 + goto l277 + l278: + position, tokenIndex = position277, tokenIndex277 if buffer[position] != rune('\\') { - goto l275 + goto l279 } position++ if buffer[position] != rune('\\') { - goto l275 + goto l279 } position++ - goto l273 - l275: - position, tokenIndex = position273, tokenIndex273 + goto l277 + l279: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\\') { + goto l280 + } + position++ + if buffer[position] != rune('n') { + goto l280 + } + position++ + goto l277 + l280: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\\') { + goto l281 + } + position++ + if buffer[position] != rune('t') { + goto l281 + } + position++ + goto l277 + l281: + position, tokenIndex = position277, tokenIndex277 { - position276, tokenIndex276 := position, tokenIndex - if buffer[position] != rune('\'') { - goto l276 + position282, tokenIndex282 := position, tokenIndex + { + position283, tokenIndex283 := position, tokenIndex + if buffer[position] != rune('\'') { + goto l284 + } + position++ + goto l283 + l284: + position, tokenIndex = position283, tokenIndex283 + if buffer[position] != rune('\\') { + goto l282 + } + position++ } - position++ - goto l272 - l276: - position, tokenIndex = position276, tokenIndex276 + l283: + goto l276 + l282: + position, tokenIndex = position282, tokenIndex282 } if !matchDot() { - goto l272 + goto l276 } } - l273: - goto l271 - l272: - position, tokenIndex = position272, tokenIndex272 + l277: + goto l275 + l276: + position, tokenIndex = position276, tokenIndex276 } - add(rulesinglequotedstring, position270) + add(rulesinglequotedstring, position274) } return true }, /* 24 fieldExpr <- <(([a-z] / [A-Z] / '_') ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position285, tokenIndex285 := position, tokenIndex { - position278 := position + position286 := position { - position279, tokenIndex279 := position, tokenIndex + position287, tokenIndex287 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l280 + goto l288 } position++ - goto l279 - l280: - position, tokenIndex = position279, tokenIndex279 + goto l287 + l288: + position, tokenIndex = position287, tokenIndex287 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l281 + goto l289 } position++ - goto l279 - l281: - position, tokenIndex = position279, tokenIndex279 + goto l287 + l289: + position, tokenIndex = position287, tokenIndex287 if buffer[position] != rune('_') { - goto l277 + goto l285 } position++ } - l279: - l282: + l287: + l290: { - position283, tokenIndex283 := position, tokenIndex + position291, tokenIndex291 := position, tokenIndex { - position284, tokenIndex284 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l285 + goto l293 } position++ - goto l284 - l285: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l293: + position, tokenIndex = position292, tokenIndex292 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l286 + goto l294 } position++ - goto l284 - l286: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l294: + position, tokenIndex = position292, tokenIndex292 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l287 + goto l295 } position++ - goto l284 - l287: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l295: + position, tokenIndex = position292, tokenIndex292 if buffer[position] != rune('_') { - goto l288 + goto l296 } position++ - goto l284 - l288: - position, tokenIndex = position284, tokenIndex284 + goto l292 + l296: + position, tokenIndex = position292, tokenIndex292 if buffer[position] != rune('-') { - goto l283 + goto l291 } position++ } - l284: - goto l282 - l283: - position, tokenIndex = position283, tokenIndex283 + l292: + goto l290 + l291: + position, tokenIndex = position291, tokenIndex291 } - add(rulefieldExpr, position278) + add(rulefieldExpr, position286) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l285: + position, tokenIndex = position285, tokenIndex285 return false }, /* 25 field <- <(<(fieldExpr / reserved)> Action53)> */ func() bool { - position289, tokenIndex289 := position, tokenIndex + position297, tokenIndex297 := position, tokenIndex { - position290 := position + position298 := position { - position291 := position + position299 := position { - position292, tokenIndex292 := position, tokenIndex + position300, tokenIndex300 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l293 + goto l301 } - goto l292 - l293: - position, tokenIndex = position292, tokenIndex292 + goto l300 + l301: + position, tokenIndex = position300, tokenIndex300 { - position294 := position + position302 := position { - position295, tokenIndex295 := position, tokenIndex + position303, tokenIndex303 := position, tokenIndex if buffer[position] != rune('_') { - goto l296 + goto l304 } position++ if buffer[position] != rune('r') { - goto l296 + goto l304 } position++ if buffer[position] != rune('o') { - goto l296 + goto l304 } position++ if buffer[position] != rune('w') { - goto l296 + goto l304 } position++ - goto l295 - l296: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l304: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l297 + goto l305 } position++ if buffer[position] != rune('c') { - goto l297 + goto l305 } position++ if buffer[position] != rune('o') { - goto l297 + goto l305 } position++ if buffer[position] != rune('l') { - goto l297 + goto l305 } position++ - goto l295 - l297: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l305: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l298 + goto l306 } position++ if buffer[position] != rune('s') { - goto l298 + goto l306 } position++ if buffer[position] != rune('t') { - goto l298 + goto l306 } position++ if buffer[position] != rune('a') { - goto l298 + goto l306 } position++ if buffer[position] != rune('r') { - goto l298 + goto l306 } position++ if buffer[position] != rune('t') { - goto l298 + goto l306 } position++ - goto l295 - l298: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l306: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l299 + goto l307 } position++ if buffer[position] != rune('e') { - goto l299 + goto l307 } position++ if buffer[position] != rune('n') { - goto l299 + goto l307 } position++ if buffer[position] != rune('d') { - goto l299 + goto l307 } position++ - goto l295 - l299: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l307: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l300 + goto l308 } position++ if buffer[position] != rune('t') { - goto l300 + goto l308 } position++ if buffer[position] != rune('i') { - goto l300 + goto l308 } position++ if buffer[position] != rune('m') { - goto l300 + goto l308 } position++ if buffer[position] != rune('e') { - goto l300 + goto l308 } position++ if buffer[position] != rune('s') { - goto l300 + goto l308 } position++ if buffer[position] != rune('t') { - goto l300 + goto l308 } position++ if buffer[position] != rune('a') { - goto l300 + goto l308 } position++ if buffer[position] != rune('m') { - goto l300 + goto l308 } position++ if buffer[position] != rune('p') { - goto l300 + goto l308 } position++ - goto l295 - l300: - position, tokenIndex = position295, tokenIndex295 + goto l303 + l308: + position, tokenIndex = position303, tokenIndex303 if buffer[position] != rune('_') { - goto l289 + goto l297 } position++ if buffer[position] != rune('f') { - goto l289 + goto l297 } position++ if buffer[position] != rune('i') { - goto l289 + goto l297 } position++ if buffer[position] != rune('e') { - goto l289 + goto l297 } position++ if buffer[position] != rune('l') { - goto l289 + goto l297 } position++ if buffer[position] != rune('d') { - goto l289 + goto l297 } position++ } - l295: - add(rulereserved, position294) + l303: + add(rulereserved, position302) } } - l292: - add(rulePegText, position291) + l300: + add(rulePegText, position299) } { add(ruleAction53, position) } - add(rulefield, position290) + add(rulefield, position298) } return true - l289: - position, tokenIndex = position289, tokenIndex289 + l297: + position, tokenIndex = position297, tokenIndex297 return false }, /* 26 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, /* 27 posfield <- <( Action54)> */ func() bool { - position303, tokenIndex303 := position, tokenIndex + position311, tokenIndex311 := position, tokenIndex { - position304 := position + position312 := position { - position305 := position + position313 := position if !_rules[rulefieldExpr]() { - goto l303 + goto l311 } - add(rulePegText, position305) + add(rulePegText, position313) } { add(ruleAction54, position) } - add(ruleposfield, position304) + add(ruleposfield, position312) } return true - l303: - position, tokenIndex = position303, tokenIndex303 + l311: + position, tokenIndex = position311, tokenIndex311 return false }, /* 28 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position307, tokenIndex307 := position, tokenIndex + position315, tokenIndex315 := position, tokenIndex { - position308 := position + position316 := position { - position309, tokenIndex309 := position, tokenIndex + position317, tokenIndex317 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l310 + goto l318 } position++ - l311: + l319: { - position312, tokenIndex312 := position, tokenIndex + position320, tokenIndex320 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l312 + goto l320 } position++ - goto l311 - l312: - position, tokenIndex = position312, tokenIndex312 + goto l319 + l320: + position, tokenIndex = position320, tokenIndex320 } - goto l309 - l310: - position, tokenIndex = position309, tokenIndex309 + goto l317 + l318: + position, tokenIndex = position317, tokenIndex317 if buffer[position] != rune('0') { - goto l307 + goto l315 } position++ } - l309: - add(ruleuint, position308) + l317: + add(ruleuint, position316) } return true - l307: - position, tokenIndex = position307, tokenIndex307 + l315: + position, tokenIndex = position315, tokenIndex315 return false }, - /* 29 col <- <(( Action55) / ('\'' '\'' Action56) / ('"' '"' Action57))> */ + /* 29 col <- <(( Action55) / (<('\'' singlequotedstring '\'')> Action56) / (<('"' doublequotedstring '"')> Action57))> */ func() bool { - position313, tokenIndex313 := position, tokenIndex + position321, tokenIndex321 := position, tokenIndex { - position314 := position + position322 := position { - position315, tokenIndex315 := position, tokenIndex + position323, tokenIndex323 := position, tokenIndex { - position317 := position + position325 := position if !_rules[ruleuint]() { - goto l316 + goto l324 } - add(rulePegText, position317) + add(rulePegText, position325) } { add(ruleAction55, position) } - goto l315 - l316: - position, tokenIndex = position315, tokenIndex315 - if buffer[position] != rune('\'') { - goto l319 - } - position++ + goto l323 + l324: + position, tokenIndex = position323, tokenIndex323 { - position320 := position - if !_rules[rulesinglequotedstring]() { - goto l319 + position328 := position + if buffer[position] != rune('\'') { + goto l327 } - add(rulePegText, position320) + position++ + if !_rules[rulesinglequotedstring]() { + goto l327 + } + if buffer[position] != rune('\'') { + goto l327 + } + position++ + add(rulePegText, position328) } - if buffer[position] != rune('\'') { - goto l319 - } - position++ { add(ruleAction56, position) } - goto l315 - l319: - position, tokenIndex = position315, tokenIndex315 - if buffer[position] != rune('"') { - goto l313 - } - position++ + goto l323 + l327: + position, tokenIndex = position323, tokenIndex323 { - position322 := position - if !_rules[ruledoublequotedstring]() { - goto l313 + position330 := position + if buffer[position] != rune('"') { + goto l321 } - add(rulePegText, position322) + position++ + if !_rules[ruledoublequotedstring]() { + goto l321 + } + if buffer[position] != rune('"') { + goto l321 + } + position++ + add(rulePegText, position330) } - if buffer[position] != rune('"') { - goto l313 - } - position++ { add(ruleAction57, position) } } - l315: - add(rulecol, position314) + l323: + add(rulecol, position322) } return true - l313: - position, tokenIndex = position313, tokenIndex313 + l321: + position, tokenIndex = position321, tokenIndex321 return false }, - /* 30 row <- <(( Action58) / ('\'' '\'' Action59) / ('"' '"' Action60))> */ + /* 30 row <- <(( Action58) / (<('\'' singlequotedstring '\'')> Action59) / (<('"' doublequotedstring '"')> Action60))> */ nil, /* 31 open <- <('(' sp)> */ func() bool { - position325, tokenIndex325 := position, tokenIndex + position333, tokenIndex333 := position, tokenIndex { - position326 := position + position334 := position if buffer[position] != rune('(') { - goto l325 + goto l333 } position++ if !_rules[rulesp]() { - goto l325 + goto l333 } - add(ruleopen, position326) + add(ruleopen, position334) } return true - l325: - position, tokenIndex = position325, tokenIndex325 + l333: + position, tokenIndex = position333, tokenIndex333 return false }, /* 32 close <- <(')' sp)> */ func() bool { - position327, tokenIndex327 := position, tokenIndex + position335, tokenIndex335 := position, tokenIndex { - position328 := position + position336 := position if buffer[position] != rune(')') { - goto l327 + goto l335 } position++ if !_rules[rulesp]() { - goto l327 + goto l335 } - add(ruleclose, position328) + add(ruleclose, position336) } return true - l327: - position, tokenIndex = position327, tokenIndex327 + l335: + position, tokenIndex = position335, tokenIndex335 return false }, /* 33 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position330 := position - l331: + position338 := position + l339: { - position332, tokenIndex332 := position, tokenIndex + position340, tokenIndex340 := position, tokenIndex { - position333, tokenIndex333 := position, tokenIndex + position341, tokenIndex341 := position, tokenIndex if buffer[position] != rune(' ') { - goto l334 + goto l342 } position++ - goto l333 - l334: - position, tokenIndex = position333, tokenIndex333 + goto l341 + l342: + position, tokenIndex = position341, tokenIndex341 if buffer[position] != rune('\t') { - goto l335 + goto l343 } position++ - goto l333 - l335: - position, tokenIndex = position333, tokenIndex333 + goto l341 + l343: + position, tokenIndex = position341, tokenIndex341 if buffer[position] != rune('\n') { - goto l332 + goto l340 } position++ } - l333: - goto l331 - l332: - position, tokenIndex = position332, tokenIndex332 + l341: + goto l339 + l340: + position, tokenIndex = position340, tokenIndex340 } - add(rulesp, position330) + add(rulesp, position338) } return true }, /* 34 comma <- <(sp ',' sp)> */ func() bool { - position336, tokenIndex336 := position, tokenIndex + position344, tokenIndex344 := position, tokenIndex { - position337 := position + position345 := position if !_rules[rulesp]() { - goto l336 + goto l344 } if buffer[position] != rune(',') { - goto l336 + goto l344 } position++ if !_rules[rulesp]() { - goto l336 + goto l344 } - add(rulecomma, position337) + add(rulecomma, position345) } return true - l336: - position, tokenIndex = position336, tokenIndex336 + l344: + position, tokenIndex = position344, tokenIndex344 return false }, /* 35 lbrack <- <('[' sp)> */ func() bool { - position338, tokenIndex338 := position, tokenIndex + position346, tokenIndex346 := position, tokenIndex { - position339 := position + position347 := position if buffer[position] != rune('[') { - goto l338 + goto l346 } position++ if !_rules[rulesp]() { - goto l338 + goto l346 } - add(rulelbrack, position339) + add(rulelbrack, position347) } return true - l338: - position, tokenIndex = position338, tokenIndex338 + l346: + position, tokenIndex = position346, tokenIndex346 return false }, /* 36 rbrack <- <(sp ']' sp)> */ func() bool { - position340, tokenIndex340 := position, tokenIndex + position348, tokenIndex348 := position, tokenIndex { - position341 := position + position349 := position if !_rules[rulesp]() { - goto l340 + goto l348 } if buffer[position] != rune(']') { - goto l340 + goto l348 } position++ if !_rules[rulesp]() { - goto l340 + goto l348 } - add(rulerbrack, position341) + add(rulerbrack, position349) } return true - l340: - position, tokenIndex = position340, tokenIndex340 + l348: + position, tokenIndex = position348, tokenIndex348 return false }, /* 37 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position342, tokenIndex342 := position, tokenIndex + position350, tokenIndex350 := position, tokenIndex { - position343 := position + position351 := position { - position344, tokenIndex344 := position, tokenIndex + position352, tokenIndex352 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l345 + goto l353 } position++ - goto l344 - l345: - position, tokenIndex = position344, tokenIndex344 + goto l352 + l353: + position, tokenIndex = position352, tokenIndex352 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l342 + goto l350 } position++ } - l344: - l346: + l352: + l354: { - position347, tokenIndex347 := position, tokenIndex + position355, tokenIndex355 := position, tokenIndex { - position348, tokenIndex348 := position, tokenIndex + position356, tokenIndex356 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l349 + goto l357 } position++ - goto l348 - l349: - position, tokenIndex = position348, tokenIndex348 + goto l356 + l357: + position, tokenIndex = position356, tokenIndex356 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l350 + goto l358 } position++ - goto l348 - l350: - position, tokenIndex = position348, tokenIndex348 + goto l356 + l358: + position, tokenIndex = position356, tokenIndex356 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l347 + goto l355 } position++ } - l348: - goto l346 - l347: - position, tokenIndex = position347, tokenIndex347 + l356: + goto l354 + l355: + position, tokenIndex = position355, tokenIndex355 } - add(ruleIDENT, position343) + add(ruleIDENT, position351) } return true - l342: - position, tokenIndex = position342, tokenIndex342 + l350: + position, tokenIndex = position350, tokenIndex350 return false }, /* 38 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position351, tokenIndex351 := position, tokenIndex + position359, tokenIndex359 := position, tokenIndex { - position352 := position + position360 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('-') { - goto l351 + goto l359 } position++ { - position353, tokenIndex353 := position, tokenIndex + position361, tokenIndex361 := position, tokenIndex if buffer[position] != rune('0') { - goto l354 + goto l362 } position++ - goto l353 - l354: - position, tokenIndex = position353, tokenIndex353 + goto l361 + l362: + position, tokenIndex = position361, tokenIndex361 if buffer[position] != rune('1') { - goto l351 + goto l359 } position++ } - l353: + l361: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('-') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune('T') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if buffer[position] != rune(':') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l351 + goto l359 } position++ - add(ruletimestampbasicfmt, position352) + add(ruletimestampbasicfmt, position360) } return true - l351: - position, tokenIndex = position351, tokenIndex351 + l359: + position, tokenIndex = position359, tokenIndex359 return false }, /* 39 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position355, tokenIndex355 := position, tokenIndex + position363, tokenIndex363 := position, tokenIndex { - position356 := position + position364 := position { - position357, tokenIndex357 := position, tokenIndex + position365, tokenIndex365 := position, tokenIndex if buffer[position] != rune('"') { - goto l358 + goto l366 } position++ { - position359 := position + position367 := position if !_rules[ruletimestampbasicfmt]() { - goto l358 + goto l366 } - add(rulePegText, position359) + add(rulePegText, position367) } if buffer[position] != rune('"') { - goto l358 + goto l366 } position++ - goto l357 - l358: - position, tokenIndex = position357, tokenIndex357 + goto l365 + l366: + position, tokenIndex = position365, tokenIndex365 if buffer[position] != rune('\'') { - goto l360 + goto l368 } position++ { - position361 := position + position369 := position if !_rules[ruletimestampbasicfmt]() { - goto l360 + goto l368 } - add(rulePegText, position361) + add(rulePegText, position369) } if buffer[position] != rune('\'') { - goto l360 + goto l368 } position++ - goto l357 - l360: - position, tokenIndex = position357, tokenIndex357 + goto l365 + l368: + position, tokenIndex = position365, tokenIndex365 { - position362 := position + position370 := position if !_rules[ruletimestampbasicfmt]() { - goto l355 + goto l363 } - add(rulePegText, position362) + add(rulePegText, position370) } } - l357: - add(ruletimestampfmt, position356) + l365: + add(ruletimestampfmt, position364) } return true - l355: - position, tokenIndex = position355, tokenIndex355 + l363: + position, tokenIndex = position363, tokenIndex363 return false }, /* 40 timestamp <- <( Action61)> */ @@ -3572,7 +3637,7 @@ func (p *PQL) Init(options ...func(*PQL) error) error { nil, /* 89 Action46 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 90 Action47 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 90 Action47 <- <{ p.addVal(buffer[begin:end]) }> */ nil, /* 91 Action48 <- <{ p.addVal(buffer[begin:end]) }> */ nil, diff --git a/server/server_test.go b/server/server_test.go index d7cb9452e..96d0fa527 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1092,6 +1092,50 @@ func TestClusterExhaustingConnections(t *testing.T) { } } +func TestQueryingWithQuotesAndStuff(t *testing.T) { + m := test.RunCommand(t) + defer m.Close() + + client, err := http.NewInternalClient(m.API.Node().URI.HostPort(), http.GetHTTPClient(nil)) + if err != nil { + t.Fatal(err) + } + + // Execute Set() commands. + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } + if err := client.CreateFieldWithOptions(context.Background(), "i", "fld", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + // Test escaped single quote gets set properly + if res, err := m.Query(t, "i", "", `Set('bl\'ah', fld=ha)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, "[true]") { + t.Errorf("setting escaped single quote result: %s", res) + } + if res, err := m.Query(t, "i", "", `Row(fld=ha)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, `bl'ah`) { + t.Errorf("value with escaped single quote set improperly: %s", res) + } + + // Test escaped double quote gets set properly + if res, err := m.Query(t, "i", "", `Set("d\"ah", fld=dq)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, "[true]") { + t.Errorf("value with escaped double quote set improperly: %s", res) + } + if res, err := m.Query(t, "i", "", `Row(fld=dq)`); err != nil { + t.Fatal(err) + } else if !strings.Contains(res, `d\"ah`) { + // the backslash is there because JSON needs to escape the + // double quote since it uses double quotes + t.Errorf("value with escaped double quote set improperly: %s", res) + } +} + func TestClusterExhaustingConnectionsImport(t *testing.T) { if !runStress { t.Skip("stress")